diff --git a/GramAddict/__main__.py b/GramAddict/__main__.py index cff8d27..c94966c 100644 --- a/GramAddict/__main__.py +++ b/GramAddict/__main__.py @@ -1,8 +1,8 @@ -from GramAddict.core.agentic_views import * import argparse from os import getcwd, path from GramAddict import __version__ +from GramAddict.core.agentic_views import * from GramAddict.core.bot_flow import start_bot from GramAddict.core.download_from_github import download_from_github @@ -13,9 +13,7 @@ def cmd_init(args): for username in args.account_name: if not path.exists("./run.py"): print("Creating run.py ...") - download_from_github( - "https://github.com/GramAddict/bot/blob/master/run.py" - ) + download_from_github("https://github.com/GramAddict/bot/blob/master/run.py") if not path.exists(f"./accounts/{username}"): print( f"Creating 'accounts/{username}' folder with a config starting point inside. You have to edit these files according with https://docs.gramaddict.org/#/configuration" @@ -53,8 +51,10 @@ def cmd_dump(args): os.popen("adb shell pkill atx-agent").close() try: d = u2.connect(args.device) - except RuntimeError as err: - raise SystemExit(err) + except Exception as err: + raise SystemExit( + f"⚠️ [ADB ConnectError] Could not connect to device: {err}\nPlease check if ADB is running and your device is authorized." + ) def dump_hierarchy(device, path): xml_dump = device.dump_hierarchy() @@ -71,11 +71,7 @@ def cmd_dump(args): dump_hierarchy(d, "dump/cur/hierarchy.xml") archive_name = int(time.time()) make_archive(archive_name) - print( - Fore.GREEN - + Style.BRIGHT - + "\nCurrent screen dump generated successfully! Please, send me this file:" - ) + print(Fore.GREEN + Style.BRIGHT + "\nCurrent screen dump generated successfully! Please, send me this file:") print(Fore.BLUE + Style.BRIGHT + f"{os.getcwd()}\\screen_{archive_name}.zip") @@ -126,9 +122,7 @@ def main() -> None: prog="GramAddict", description="free human-like Instagram bot", ) - parser.add_argument( - "-v", "--version", action="version", version=f"{parser.prog} {__version__}" - ) + parser.add_argument("-v", "--version", action="version", version=f"{parser.prog} {__version__}") subparser = parser.add_subparsers(dest="subparser") actions = {} for c in _commands: diff --git a/GramAddict/core/bot_flow.py b/GramAddict/core/bot_flow.py index a009a43..bd35e42 100644 --- a/GramAddict/core/bot_flow.py +++ b/GramAddict/core/bot_flow.py @@ -849,6 +849,7 @@ def _run_zero_latency_feed_loop( consecutive_marker_misses += 1 if consecutive_marker_misses >= 3: logger.error("❌ Lost context completely. Aborting feed loop to force reset.") + sae.unlearn_current_state(context_xml) dump_ui_state(device, "context_lost", {"feed": job_target, "misses": consecutive_marker_misses}) return "CONTEXT_LOST" @@ -888,6 +889,7 @@ def _run_zero_latency_feed_loop( consecutive_marker_misses += 1 if consecutive_marker_misses >= 3: logger.error("❌ Lost context completely. Aborting feed loop to force reset.") + sae.unlearn_current_state(context_xml) dump_ui_state(device, "context_lost", {"feed": job_target, "misses": consecutive_marker_misses}) return "CONTEXT_LOST" diff --git a/GramAddict/core/device_facade.py b/GramAddict/core/device_facade.py index 9521176..4a93019 100644 --- a/GramAddict/core/device_facade.py +++ b/GramAddict/core/device_facade.py @@ -1,19 +1,17 @@ import logging -import json import os -import re -import uiautomator2 as u2 -from time import sleep, time -from random import uniform -from GramAddict.core.utils import random_sleep from functools import wraps +from random import uniform +from time import sleep -from GramAddict.core.physics.biomechanics import PhysicsBody, BezierGesture +import uiautomator2 as u2 + +from GramAddict.core.physics.biomechanics import BezierGesture, PhysicsBody from GramAddict.core.physics.sendevent_injector import SendEventInjector - logger = logging.getLogger(__name__) + def adb_retry(retries=3, delay=2.0): def decorator(func): @wraps(func) @@ -28,18 +26,38 @@ def adb_retry(retries=3, delay=2.0): sleep(delay * (attempt + 1)) # Exponential backoff logger.error(f"❌ ADB action {func.__name__} failed after {retries} retries. Crashing gracefully.") raise last_err + return wrapper + return decorator + def create_device(device_id, app_id, args=None): try: return DeviceFacade(device_id, app_id, args) except Exception as e: + err_str = str(e) + err_type = str(type(e)) + if ( + "ConnectError" in err_type + or "ConnectionRefusedError" in err_type + or "ConnectionError" in err_type + or "Timeout" in err_type + ): + logger.error(f"⚠️ [ADB ConnectError] Could not connect to device '{device_id}'.") + logger.error("👉 Please verify:") + logger.error(" 1. Your phone is connected via USB or Wi-Fi.") + logger.error(" 2. 'USB Debugging' is enabled in Developer Options.") + logger.error(" 3. You have authorized this computer on your phone's screen.") + logger.error(" 4. The adb server is running ('adb devices').") + raise SystemExit(1) + logger.error(f"Failed to create device: {e}") - # We don't want to just return None and crash later. + # We don't want to just return None and crash later. # We should raise so the orchestrator knows it's a fatal boot error. raise e + def get_device_info(device): if not device or not device.deviceV2: logger.error("Cannot get device info: Device not initialized.") @@ -47,32 +65,29 @@ def get_device_info(device): info = device.info logger.debug(f"Device Info: {info.get('productName')} | SDK: {info.get('sdkInt')}") + class DeviceFacade: deviceV2 = None app_id = None device_id = None - + def __init__(self, device_id, app_id, args): self.device_id = device_id self.app_id = app_id self.args = args self.deviceV2 = u2.connect(device_id) - + # Configure uiautomator2 self.deviceV2.settings["wait_timeout"] = 3.0 self.deviceV2.settings["post_delay"] = 0.5 - + # System dialog handler (language-agnostic via resource-id, not text) try: # u2 v3.x: named watchers with xpath selectors # android:id/aerr_close = App crash "Close" button (all languages) - self.deviceV2.watcher("crash_dialog").when( - xpath='//*[@resource-id="android:id/aerr_close"]' - ).click() + self.deviceV2.watcher("crash_dialog").when(xpath='//*[@resource-id="android:id/aerr_close"]').click() # android:id/button1 = positive system dialog button (all languages) - self.deviceV2.watcher("system_dialog").when( - xpath='//*[@resource-id="android:id/button1"]' - ).click() + self.deviceV2.watcher("system_dialog").when(xpath='//*[@resource-id="android:id/button1"]').click() self.deviceV2.watcher.start() except Exception as e: logger.debug(f"Could not start system watcher: {e}") @@ -88,7 +103,7 @@ class DeviceFacade: @adb_retry() def cm_to_pixels(self, cm: float) -> int: info = self.deviceV2.info - dpx = info.get("displaySizeDpX", 400) + dpx = info.get("displaySizeDpX", 400) width = info.get("displayWidth", 1080) # Android baseline: 1 dp = 1/160 inch. 1 inch = 2.54 cm # PPCM (Pixels Per CM) = (width / dpx) * (160 / 2.54) @@ -106,7 +121,6 @@ class DeviceFacade: def unlock(self): self.deviceV2.unlock() - @property def info(self): return self.deviceV2.info @@ -132,7 +146,8 @@ class DeviceFacade: def swipe(self, sx, sy, ex, ey, duration=None): """Pass-through strictly for non-biological bezier swiping (e.g., darwin_engine noise correction)""" kwargs = {} - if duration is not None: kwargs["duration"] = duration + if duration is not None: + kwargs["duration"] = duration self.deviceV2.swipe(sx, sy, ex, ey, **kwargs) @adb_retry() @@ -146,14 +161,14 @@ class DeviceFacade: @adb_retry() def click(self, x=None, y=None, obj=None): if obj: - if isinstance(obj, dict) and 'x' in obj and 'y' in obj: - self.human_click(obj['x'], obj['y']) + if isinstance(obj, dict) and "x" in obj and "y" in obj: + self.human_click(obj["x"], obj["y"]) return try: left, top, right, bottom = obj.bounds() w = right - left h = bottom - top - + # Biological fingerprint via PhysicsBody body = PhysicsBody.get_session_instance(self) # Thumb bias: right-handers land slightly left-below center @@ -163,20 +178,21 @@ class DeviceFacade: else: cx_base = left + (w * 0.55) cy_base = top + (h * 0.55) - + from random import gauss + # Fatigue increases spread fatigue_mult = 1.0 + body.fatigue * 0.3 sigma_x = max(1, w * 0.15 * fatigue_mult) sigma_y = max(1, h * 0.15 * fatigue_mult) - + cx = int(gauss(cx_base, sigma_x)) cy = int(gauss(cy_base, sigma_y)) - + # Math constraint to ensure it physically lands on the button cx = max(left + 1, min(cx, right - 1)) cy = max(top + 1, min(cy, bottom - 1)) - + self.human_click(cx, cy) except Exception as e: logger.debug(f"Bounds extraction failed, fallback to native click: {e}") @@ -193,7 +209,6 @@ class DeviceFacade: self.deviceV2.shell(f"input tap {int(x)} {int(y)}") return - from random import uniform try: body = PhysicsBody.get_session_instance(self) injector = SendEventInjector.get_instance(self) @@ -201,7 +216,6 @@ class DeviceFacade: tap_duration = uniform(40, 90) timing = BezierGesture.compute_sigmoid_timing(len(points), tap_duration) - injector.inject_gesture(points, timing, touch_major=body.get_touch_major()) except Exception as e: logger.debug(f"human_click biomechanics failed, fallback: {e}") @@ -234,18 +248,17 @@ class DeviceFacade: try: body = PhysicsBody.get_session_instance(self) injector = SendEventInjector.get_instance(self) - + # Use scroll_curve for vertical swipes, horizontal_swipe_curve for horizontal is_horizontal = abs(end_x - start_x) > abs(end_y - start_y) if is_horizontal: points = BezierGesture.horizontal_swipe_curve((start_x, start_y), (end_x, end_y), body) else: points = BezierGesture.scroll_curve((start_x, start_y), (end_x, end_y), body) - + # Use fling timing (J-curve) to ensure high terminal velocity so Android scroll physics works natively timing = BezierGesture.compute_fling_timing(len(points), dur_ms) - injector.inject_gesture(points, timing, touch_major=body.get_touch_major()) except Exception as e: logger.debug(f"human_swipe biomechanics failed, fallback to native swipe: {e}") @@ -263,14 +276,14 @@ class DeviceFacade: pkg = self.deviceV2.app_current().get("package") if pkg == self.app_id: return pkg - + # Brief retry: many false positives come from <500ms notification banners # A single short wait handles ALL transient overlays regardless of source app sleep(0.5) pkg = self.deviceV2.app_current().get("package") - + # If still not our app, check if it's just SystemUI (always present, never a real takeover) - if pkg in ('com.android.systemui', 'android'): + if pkg in ("com.android.systemui", "android"): return self.app_id return pkg @@ -284,38 +297,41 @@ class DeviceFacade: def dump_hierarchy(self): # Compressed=True dramatically speeds up UIAutomator2 dumps by skipping invisible elements! xml = self.deviceV2.dump_hierarchy(compressed=True) - + # Continuous Session Tracing from datetime import datetime + try: if not hasattr(self, "_trace_counter"): self._trace_counter = 0 ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") self._trace_dir = os.path.join("debug", "session_traces", ts) os.makedirs(self._trace_dir, exist_ok=True) - + self._trace_counter += 1 trace_path = os.path.join(self._trace_dir, f"{self._trace_counter:05d}.xml") with open(trace_path, "w", encoding="utf-8") as f: f.write(xml) except Exception as e: logger.debug(f"Failed to write session trace: {e}") - + return xml @adb_retry() def get_screenshot_b64(self): import base64 from io import BytesIO + img = self.deviceV2.screenshot() buffered = BytesIO() - img.save(buffered, format="JPEG", quality=70) # Compressed for target latency - return base64.b64encode(buffered.getvalue()).decode('utf-8') + img.save(buffered, format="JPEG", quality=70) # Compressed for target latency + return base64.b64encode(buffered.getvalue()).decode("utf-8") # Telepathic Semantic UI Integration @adb_retry() def find_semantic(self, intent_description: str): from GramAddict.core.telepathic_engine import TelepathicEngine + engine = TelepathicEngine.get_instance() xml = self.dump_hierarchy() # Passing self (DeviceFacade) enables the Vision Cortex VLM fallback diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index 7386c6b..6905cb3 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -13,858 +13,25 @@ Like a GPS navigation system: - It remembers shortcuts (learn) """ -import hashlib import logging -import re import time -import xml.etree.ElementTree as ET -from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List -from GramAddict.core.qdrant_memory import QdrantBase from GramAddict.core.utils import random_sleep logger = logging.getLogger(__name__) -# ══════════════════════════════════════════════════════ -# 1. SCREEN IDENTITY — "Where am I?" -# ══════════════════════════════════════════════════════ - - -class ScreenType(Enum): - HOME_FEED = "home_feed" - EXPLORE_GRID = "explore_grid" - REELS_FEED = "reels_feed" - OWN_PROFILE = "own_profile" - OTHER_PROFILE = "other_profile" - POST_DETAIL = "post_detail" - STORY_VIEW = "story_view" - DM_INBOX = "dm_inbox" - DM_THREAD = "dm_thread" - SEARCH_RESULTS = "search_results" - FOLLOW_LIST = "follow_list" - COMMENTS = "comments" - MODAL = "modal" - FOREIGN_APP = "foreign_app" - UNKNOWN = "unknown" - - -class ScreenIdentity: - """ - Understands what screen the bot is on by analyzing the XML dump. - NO hardcoded states — purely structural analysis. - - This is the bot's EYES. It answers: "What do I see right now?" - """ - - def __init__(self, bot_username: str): - self.bot_username = bot_username.lower() - try: - from GramAddict.core.qdrant_memory import ScreenMemoryDB - - self.screen_memory = ScreenMemoryDB() - except ImportError: - self.screen_memory = None - - def identify(self, xml_dump: str) -> Dict[str, Any]: - """ - Analyzes an XML dump and returns a complete screen description. - - Returns: - { - 'screen_type': ScreenType, - 'available_actions': ['tap like button', 'tap explore tab', ...], - 'selected_tab': 'feed_tab' | 'search_tab' | ..., - 'context': {'username': '...', 'post_count': '...', ...} - } - """ - if not xml_dump or not isinstance(xml_dump, str): - return self._empty_screen() - - try: - clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip() - root = ET.fromstring(clean) - except Exception: - return self._empty_screen() - - # Extract structural signals - packages = set() - resource_ids = set() - content_descs = [] - texts = [] - selected_tab = None - clickable_elements = [] - - app_id = "com.instagram.android" - - for elem in root.iter("node"): - pkg = elem.get("package", "") - if pkg: - packages.add(pkg) - - rid = elem.get("resource-id", "").strip() - text = elem.get("text", "").strip() - desc = elem.get("content-desc", "").strip() - clickable = elem.get("clickable", "false") == "true" - selected = elem.get("selected", "false") == "true" - bounds = elem.get("bounds", "") - - if rid: - # Normalize: "com.instagram.android:id/feed_tab" → "feed_tab" - short_id = rid.split("/")[-1] if "/" in rid else rid - resource_ids.add(short_id) - - # Track which tab is selected - if selected and short_id in ("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab"): - selected_tab = short_id - - if text: - texts.append(text) - if desc: - content_descs.append(desc) - - if clickable and bounds: - match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds) - if match: - left, t, r, b = map(int, match.groups()) - cx, cy = (left + r) // 2, (t + b) // 2 - clickable_elements.append( - { - "text": text, - "desc": desc, - "id": rid.split("/")[-1] if "/" in rid else rid, - "x": cx, - "y": cy, - "bounds": bounds, - } - ) - - # ── Foreign app check ── - if app_id not in packages: - return { - "screen_type": ScreenType.FOREIGN_APP, - "available_actions": ["press back", "force start instagram"], - "selected_tab": None, - "context": {"packages": list(packages)}, - "signature": self._compute_signature(resource_ids, content_descs, texts), - } - - desc_lower = " ".join(content_descs).lower() - text_lower = " ".join(texts).lower() - ids_str = " ".join(resource_ids).lower() - - signature = self._compute_signature(resource_ids, content_descs, texts) - - # ── Identify screen type from structural signals ── - screen_type = self._classify_screen( - resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature - ) - - # ── Extract available actions from clickable elements ── - available_actions = self._extract_available_actions( - clickable_elements, resource_ids, content_descs, texts, screen_type - ) - - # ── Extract context ── - context = self._extract_context(content_descs, texts, resource_ids, screen_type) - - return { - "screen_type": screen_type, - "available_actions": available_actions, - "selected_tab": selected_tab, - "context": context, - "signature": signature, - } - - def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None): - """Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states.""" - - # Priority 0: Content-creation overlays that block ALL navigation. - # These full-screen Instagram UIs have no navigation tabs and trap the bot. - # Structural detection is O(1), zero LLM calls, and cannot be fooled. - creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera") - if any(marker in ids_str for marker in creation_flow_markers): - logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL") - return ScreenType.MODAL - - # Priority 1: Check Qdrant Semantic Cache - if signature and self.screen_memory and self.screen_memory.is_connected: - cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92) - if cached_type_str: - try: - return ScreenType[cached_type_str] - except KeyError: - pass - - # Priority 2: Structural Heuristics (Instant, for core tabs) - if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids: - return ScreenType.FOLLOW_LIST - - if "profile_header_container" in ids: - return ScreenType.OTHER_PROFILE - - # Reels structural markers — present even when Instagram hides the tab bar - # in full-screen Reels viewing. Without this, selected_tab=None → UNKNOWN. - REELS_MARKERS = ("clips_viewer_container", "root_clips_layout", "clips_linear_layout_container") - if any(marker in ids for marker in REELS_MARKERS): - return ScreenType.REELS_FEED - - # DM thread detection — structural markers present inside DM conversations - if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids: - return ScreenType.DM_THREAD - - if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab: - return ScreenType.POST_DETAIL - - if selected_tab == "feed_tab": - return ScreenType.HOME_FEED - if selected_tab == "clips_tab": - return ScreenType.REELS_FEED - if selected_tab == "search_tab": - return ScreenType.EXPLORE_GRID - if selected_tab == "profile_tab": - return ScreenType.OWN_PROFILE - if selected_tab == "direct_tab": - return ScreenType.DM_INBOX - if "message_input" in ids: - return ScreenType.DM_INBOX # Fallback for DM thread as inbox - - # Priority 3: Semantic VLM Classification Fallback - from GramAddict.core.config import Config - from GramAddict.core.llm_provider import query_llm - - cfg = Config() - url = ( - getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat") - if hasattr(cfg, "args") - else "http://localhost:11434/api/chat" - ) - model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3" - - layout_context = ( - f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n" - ) - prompt = ( - f"Identify the Instagram screen layout type based on these DOM structural signals.\n" - f"Valid types: {[t.name for t in ScreenType]}\n" - f"Context:\n{layout_context}\n" - f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches." - ) - - try: - response = query_llm( - url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False - ) - if response and isinstance(response, str): - result = response.strip().upper() - elif response and isinstance(response, dict) and "response" in response: - result = response["response"].strip().upper() - else: - return ScreenType.UNKNOWN - - for t in ScreenType: - if t.name in result: - if signature and self.screen_memory: - self.screen_memory.store_screen(signature, t.name) - return t - except Exception as e: - import logging - - logging.getLogger(__name__).debug(f"LLM Classification failed: {e}") - - return ScreenType.UNKNOWN - - def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type): - """Discover what actions are possible on this screen.""" - actions = [] - - # Navigation tabs (always available when visible) - tab_map = { - "feed_tab": "tap home tab", - "search_tab": "tap explore tab", - "clips_tab": "tap reels tab", - "profile_tab": "tap profile tab", - "direct_tab": "tap messages tab", - } - for tab_id, action in tab_map.items(): - if tab_id in resource_ids: - actions.append(action) - - # Screen-specific actions - desc_lower = " ".join(content_descs).lower() - text_lower = " ".join(texts).lower() - - if "like" in desc_lower: - actions.append("tap like button") - if "comment" in desc_lower: - actions.append("tap comment button") - if "share" in desc_lower: - actions.append("tap share button") - if "save" in desc_lower or "bookmark" in desc_lower: - actions.append("tap save button") - if "back" in desc_lower: - actions.append("tap back button") - if any("follow" in e.get("text", "").lower() for e in clickable_elements): - actions.append("tap follow button") - - if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE: - if "message" in desc_lower or "nachricht" in desc_lower: - actions.append("tap message button") - if ( - "following" in desc_lower - or "abonniert" in desc_lower - or "following" in text_lower - or "profile_header_following" in " ".join(resource_ids).lower() - ): - actions.append("tap following list") - - # Grid items - if screen_type == ScreenType.EXPLORE_GRID: - actions.append("tap first grid item") - - # Scroll - actions.append("scroll down") - actions.append("press back") - - return list(set(actions)) # Deduplicate - - def _extract_context(self, content_descs, texts, resource_ids, screen_type): - """Extract meaningful context from the screen.""" - context = {} - - desc_text = " ".join(content_descs) - - # Username on profile - username_match = re.search(r"(\w+)'s (?:profile|story|unseen story)", desc_text) - if username_match: - context["username"] = username_match.group(1) - - # Post/follower counts - for d in content_descs: - m = re.match(r"([\d,.]+K?M?)(\s*)(posts?|followers?|following)", d, re.IGNORECASE) - if m: - context[m.group(3).lower()] = m.group(1) - - # Like state - for d in content_descs: - if d.lower() == "liked": - context["is_liked"] = True - elif d.lower() == "like": - context["is_liked"] = False - - return context - - def _compute_signature(self, resource_ids, content_descs, texts): - """Compute a stable hash for this screen state (for Qdrant lookup).""" - # Use sorted IDs + key content for stability - sig_parts = sorted(resource_ids)[:20] - sig_parts.extend(sorted(set(d.lower()[:30] for d in content_descs if len(d) > 2))[:10]) - sig = "|".join(sig_parts) - return hashlib.sha256(sig.encode()).hexdigest()[:24] - - def _empty_screen(self): - return { - "screen_type": ScreenType.FOREIGN_APP, - "available_actions": ["press back", "force start instagram"], - "selected_tab": None, - "context": {}, - "signature": "empty", - } +from GramAddict.core.navigation.knowledge import NavigationKnowledge +from GramAddict.core.navigation.path_memory import PathMemory +from GramAddict.core.navigation.planner import GoalPlanner +from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType +# Re-export for backward compatibility (optional but helps minimize import breakage) +__all__ = ["GoalExecutor", "ScreenIdentity", "ScreenType", "PathMemory", "NavigationKnowledge", "GoalPlanner"] # ══════════════════════════════════════════════════════ -# 2. PATH MEMORY — "How did I get there last time?" -# ══════════════════════════════════════════════════════ - - -class PathMemory: - """ - Qdrant-backed memory for successful navigation paths. - - Stores: goal → [step1, step2, ...] → success - Enables instant recall for known goals. - """ - - def __init__(self, username: str = ""): - self.username = username - try: - suffix = f"_{username}" if username else "" - self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768) - except Exception: - self._db = None - - def wipe(self): - """Wipe all learned navigation paths from Qdrant.""" - if self._db and self._db.is_connected: - try: - self._db.wipe_collection() - except Exception as e: - logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}") - - def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]: - """ - Recall a previously successful path for this goal from this screen type. - Returns list of steps or None. - """ - if not self._db or not self._db.is_connected: - return None - - query = f"goal: {goal} | from: {current_screen_type}" - vec = self._db._get_embedding(query) - if not vec: - return None - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.query_points( - collection_name=self._db.collection_name, - query=vec, - query_filter=Filter( - must=[FieldCondition(key="start_screen", match=MatchValue(value=current_screen_type))] - ), - limit=3, - score_threshold=0.85, - ).points - - for r in results: - p = r.payload - if p.get("success") and p.get("steps"): - logger.info( - f"🧠 [GOAP Recall] Found path for '{goal}': " - f"{len(p['steps'])} steps (confidence: {p.get('confidence', 0):.2f})" - ) - return p["steps"] - - return None - except Exception as e: - logger.debug(f"GOAP recall error: {e}") - return None - - def learn_path(self, goal: str, start_screen: str, steps: List[Dict], success: bool): - """Store a navigation path in Qdrant.""" - if not self._db or not self._db.is_connected: - return - - query = f"goal: {goal} | from: {start_screen}" - vec = self._db._get_embedding(query) - if not vec: - return - - seed = f"{goal}|{start_screen}" - payload = { - "goal": goal, - "start_screen": start_screen, - "steps": steps, - "step_count": len(steps), - "success": success, - "confidence": 0.85 if success else 0.0, - "timestamp": time.time(), - } - - outcome = "✅" if success else "❌" - self._db.upsert_point( - seed, - payload, - vector=vec, - log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}", - ) - - def forget_path(self, goal: str, start_screen: str): - """Remove a cached path to force re-discovery.""" - if not self._db or not self._db.is_connected: - return - - seed = f"{goal}|{start_screen}" - try: - from qdrant_client import models - - point_id = self._db._get_id(seed) - self._db.client.delete( - collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id]) - ) - except Exception as e: - logger.debug(f"Failed to forget path: {e}") - - -# ══════════════════════════════════════════════════════ -# 3. GOAL PLANNER — "What should I do next?" -# ══════════════════════════════════════════════════════ - - -class NavigationKnowledge: - """ - Manages the bot's learned understanding of the Instagram UI. - Discovered dynamically through exploration and success. - """ - - def __init__(self, username: str): - self.username = username - try: - self._db = QdrantBase("navigation_knowledge", vector_size=768) - except Exception: - self._db = None - - # In-memory cache for rapidly avoiding traps during exploration - # In-memory cache for rapidly avoiding traps during exploration - self._learned_screen_mappings = {} - self._learned_traps = set() - - def wipe(self): - """Wipe all learned knowledge from Qdrant.""" - if self._db and self._db.is_connected: - try: - self._db.wipe_collection() - except Exception as e: - logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}") - - def update_username(self, username: str): - """Update username and reconnect DB if needed.""" - if self.username != username: - self.username = username - try: - self._db = QdrantBase("navigation_knowledge", vector_size=768) - except Exception: - self._db = None - - def get_requirements(self, goal: str) -> List[ScreenType]: - """Get required screens for a goal. Returns known requirements or empty list.""" - if not self._db or not self._db.is_connected: - return [] - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.scroll( - collection_name=self._db.collection_name, - scroll_filter=Filter(must=[FieldCondition(key="goal", match=MatchValue(value=goal))]), - limit=1, - )[0] - if results: - screen_name = results[0].payload.get("required_screen") - logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}") - if screen_name: - return [ScreenType[screen_name]] - except Exception as e: - logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}") - return [] - - def learn_goal_requirement(self, goal: str, screen_type: ScreenType): - """Learn that achieving 'goal' lands us on 'screen_type'.""" - if not self._db or not self._db.is_connected: - logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected") - return - - seed = f"req_{goal}" - vec = self._db._get_embedding(f"goal_requirement: {goal}") - payload = {"goal": goal, "required_screen": screen_type.name, "timestamp": time.time()} - self._db.upsert_point(seed, payload, vector=vec) - logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}' → {screen_type.name}") - - def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]: - """Find which action leads to this screen.""" - for action, screen in self._learned_screen_mappings.items(): - if screen == target_screen: - return action - - if not self._db or not self._db.is_connected: - return None - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.scroll( - collection_name=self._db.collection_name, - scroll_filter=Filter( - must=[FieldCondition(key="result_screen", match=MatchValue(value=target_screen.name))] - ), - limit=1, - )[0] - if results: - return results[0].payload.get("action") - except Exception: - pass - return None - - def get_screen_for_action(self, action: str) -> Optional[ScreenType]: - """Find where this action leads to to avoid looping traps.""" - if action in self._learned_screen_mappings: - return self._learned_screen_mappings[action] - - if not self._db or not self._db.is_connected: - return None - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.scroll( - collection_name=self._db.collection_name, - scroll_filter=Filter(must=[FieldCondition(key="action", match=MatchValue(value=action))]), - limit=1, - )[0] - if results: - screen_name = results[0].payload.get("result_screen") - if screen_name: - return ScreenType[screen_name] - except Exception: - pass - return None - - def learn_screen_mapping(self, action: str, result_screen: ScreenType): - """Learn that taking 'action' leads to 'result_screen'.""" - if not self._db or not self._db.is_connected: - return - - seed = f"map_{action}" - vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}") - payload = {"action": action, "result_screen": result_screen.name, "timestamp": time.time()} - - self._learned_screen_mappings[action] = result_screen - - self._db.upsert_point(seed, payload, vector=vec) - logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}' → {result_screen.name}") - - def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]: - """Find where this tab leads to to avoid looping traps.""" - if tab_id in self._learned_screen_mappings: - return self._learned_screen_mappings[tab_id] - - if not self._db or not self._db.is_connected: - return None - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.scroll( - collection_name=self._db.collection_name, - scroll_filter=Filter(must=[FieldCondition(key="tab_id", match=MatchValue(value=tab_id))]), - limit=1, - )[0] - if results: - s_name = results[0].payload.get("result_screen") - if s_name: - return ScreenType[s_name] - except Exception: - pass - return None - - def learn_trap(self, screen_type: ScreenType, action: str, trap_reason: str = "softlock"): - """Aversively learn that an action on a screen is dangerous/useless.""" - trap_key = f"{screen_type.name}_{action}" - self._learned_traps.add(trap_key) - - if not self._db or not self._db.is_connected: - return - - seed = f"trap_{trap_key}" - # Aversive vector is completely orthogonal to normal goals to prevent retrieval overlap - vec = self._db._get_embedding(f"trap_avoidance: {trap_key} {trap_reason}") - payload = { - "trap_screen": screen_type.name, - "trap_action": action, - "trap_reason": trap_reason, - "timestamp": time.time(), - } - self._db.upsert_point(seed, payload, vector=vec) - logger.error(f"💀 [Aversive Learning] BURNED action '{action}' on {screen_type.name} due to: {trap_reason}") - - def is_trap(self, screen_type: ScreenType, action: str) -> bool: - """Check if an action on this screen is a known trap.""" - trap_key = f"{screen_type.name}_{action}" - if trap_key in self._learned_traps: - return True - - if not self._db or not self._db.is_connected: - return False - - try: - from qdrant_client.models import FieldCondition, Filter, MatchValue - - results = self._db.client.scroll( - collection_name=self._db.collection_name, - scroll_filter=Filter( - must=[ - FieldCondition(key="trap_screen", match=MatchValue(value=screen_type.name)), - FieldCondition(key="trap_action", match=MatchValue(value=action)), - ] - ), - limit=1, - )[0] - if results: - self._learned_traps.add(trap_key) - return True - except Exception: - pass - return False - - -class GoalPlanner: - """ - Given a goal and current screen state, plans the next action. - - Uses Dynamic Discovery to navigate without hardcoded maps. - """ - - def __init__(self, username: str): - self.knowledge = NavigationKnowledge(username) - - def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]: - """Plans the NEXT single action to take toward the goal.""" - screen_type = screen["screen_type"] - available = screen.get("available_actions", []) - context = screen.get("context", {}) - goal_lower = goal.lower() - - # ── 1. Check if goal is ALREADY achieved ── - if self._is_goal_achieved(goal_lower, screen_type, context): - logger.info(f"🎯 [GOAP] Goal '{goal}' already achieved on {screen_type.value}!") - return None - - # (Phase 5: legacy _plan_goal_action static heuristics purged, - # all intents fall through to VLM-driven Discovery in _plan_navigation) - - # ── 3. Am I on the right screen? If not, navigate there ── - selected_tab = screen.get("selected_tab") - nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions) - if nav_action: - return nav_action - - # Final fallback: back-track, UNLESS back-tracking is a known trap on this screen! - if not self.knowledge.is_trap(screen_type, "press back"): - return "press back" - - # We are trapped! Can't go forward, can't go back! - logger.error(f"💀 [GOAP] Completely trapped on {screen_type.name}. Forcing Instagram restart.") - return "force start instagram" - - def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool: - """Check if the goal is already satisfied. Delegates to ScreenTopology SSOT.""" - from GramAddict.core.screen_topology import ScreenTopology - - # Interaction goals (context-specific, not navigation) - if "like" in goal and context.get("is_liked") is True: - return True - if "view profile" in goal and screen_type in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE): - return True - - # Navigation goals — delegate to SSOT - target = ScreenTopology.goal_to_target_screen(goal) - if target and screen_type == target: - return True - - return False - - def _plan_navigation( - self, - goal: str, - screen_type: ScreenType, - available: List[str], - selected_tab: Optional[str] = None, - explored_nav_actions: set = None, - ) -> Optional[str]: - """If we're on the wrong screen, figure out how to navigate. - - Strategy (priority order): - 1. HD Map (ScreenTopology BFS) — deterministic, pre-computed routes - 2. Learned Knowledge (Qdrant) — dynamic discovery from past sessions - 3. Autonomous Discovery — linguistic matching + VLM intent - """ - from GramAddict.core.screen_topology import ScreenTopology - - # 0. Aversive Filter: Remove known traps from available actions - safe_available = [] - for action in available: - if not self.knowledge.is_trap(screen_type, action): - safe_available.append(action) - else: - logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'") - available = safe_available - - # ── 1. HD Map Routing (Primary Strategy) ── - target_screen = ScreenTopology.goal_to_target_screen(goal) - if target_screen and target_screen != screen_type: - route = ScreenTopology.find_route(screen_type, target_screen) - if route: - next_action, next_screen = route[0] - # Verify action isn't explored/trapped - if next_action not in (explored_nav_actions or set()): - if not self.knowledge.is_trap(screen_type, next_action): - route_desc = " → ".join(s.name for _, s in route) - logger.info( - f"🗺️ [HD Map] Route: {screen_type.name} → {route_desc}. " f"Next action: '{next_action}'" - ) - return next_action - else: - logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.") - else: - logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.") - - # ── 2. Learned Knowledge (Qdrant) ── - required_screens = self.knowledge.get_requirements(goal) - - # ── 3. Autonomous Discovery (Blank Start fallback) ── - if not required_screens: - logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.") - - # Return raw intent for TelepathicEngine discovery (VLM) - if explored_nav_actions and goal in explored_nav_actions: - logger.info( - f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking." - ) - return None # Don't return goal again — force fallback to press back - else: - return goal - - # 4. If we're already on an acceptable screen, no navigation needed - if screen_type in required_screens: - return None - - # 5. Find the action we need to take (from learned knowledge or HD map) - for target_screen in required_screens: - # Try HD Map first! - route = ScreenTopology.find_route(screen_type, target_screen) - if route: - next_action, next_screen = route[0] - if next_action not in (explored_nav_actions or set()): - if not self.knowledge.is_trap(screen_type, next_action): - logger.info(f"🧭 [Nav HD Map] Routing to required {target_screen.name} via '{next_action}'") - return next_action - - known_action = self.knowledge.get_action_for_screen(target_screen) - - if not known_action: - logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...") - - screen_friendly_name = target_screen.name.replace("_", " ").lower() - goal_words = [w.rstrip("s") for w in screen_friendly_name.split() if len(w) > 3] - - for action in available: - if any(w in action.lower() for w in goal_words): - known_target = self.knowledge.get_screen_for_action(action) - if known_target and known_target != target_screen: - continue - - logger.info( - f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'" - ) - return action - - return f"navigate to {screen_friendly_name}" - else: - if known_action in available: - logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'") - return known_action - - # If no targeted navigation works, try going back first - if "press back" in available: - return "press back" - - return None - - -# ══════════════════════════════════════════════════════ -# 4. GOAL EXECUTOR — The Main Brain Loop +# GOAL EXECUTOR — The Main Brain Loop # ══════════════════════════════════════════════════════ diff --git a/GramAddict/core/llm_provider.py b/GramAddict/core/llm_provider.py index c6f8209..7f925f4 100644 --- a/GramAddict/core/llm_provider.py +++ b/GramAddict/core/llm_provider.py @@ -1,71 +1,77 @@ -import re -import os -import json -import requests import logging -from typing import Optional, List, Dict +import os +import re +from typing import List, Optional + +import requests try: from dotenv import load_dotenv + load_dotenv() except ImportError: pass logger = logging.getLogger(__name__) + def extract_json(text: str) -> Optional[str]: """ - Robustly extracts the first JSON object or array from a string that may contain + Robustly extracts the first JSON object or array from a string that may contain natural language prefix/suffix. Also purges blocks and markdown ticks. """ if not text: return None - + # 100% Autonomous: Scrub model's internal thinking process if "" in text: - text = re.sub(r'.*?', '', text, flags=re.DOTALL).strip() + text = re.sub(r".*?", "", text, flags=re.DOTALL).strip() logger.debug("🧠 [LLM] Scoped thinking block detected and purged.") # Remove markdown code block formats - text = re.sub(r'^```json\s*', '', text, flags=re.MULTILINE) - text = re.sub(r'^```\s*', '', text, flags=re.MULTILINE) + text = re.sub(r"^```json\s*", "", text, flags=re.MULTILINE) + text = re.sub(r"^```\s*", "", text, flags=re.MULTILINE) # Try perfect json block extraction first - match = re.search(r'(\{.*\}|\[.*\])', text, re.DOTALL) + match = re.search(r"(\{.*\}|\[.*\])", text, re.DOTALL) if match: candidate = match.group(0) try: import json + json.loads(candidate) return candidate except Exception: pass - + # Smart Fallback: Truncated JSON Healing - # If standard validation fails (e.g., due to EOF truncation by local models), - # run a regex extraction pass over the raw generated text to safely salvage + # If standard validation fails (e.g., due to EOF truncation by local models), + # run a regex extraction pass over the raw generated text to safely salvage # all key-value pairs that *were* successfully completed before the truncation. import json + matches = re.findall(r'"([a-zA-Z0-9_]+)"\s*:\s*(?:([0-9.-]+)|"([^"\\]*(?:\\.[^"\\]*)*)")', text) if matches: res = {} for k, num, obj in matches: if num: try: - res[k] = float(num) if '.' in num else int(num) + res[k] = float(num) if "." in num else int(num) except ValueError: res[k] = num else: res[k] = obj.replace('\\"', '"') - + recovered_json = json.dumps(res) logger.warning(f"🔧 [Fuzzy Parse] Successfully salvaged {len(res)} keys from heavily truncated LLM output.") return recovered_json - + return None + _MODEL_PRICING_CACHE = None + def get_model_pricing(model_id: str) -> dict: global _MODEL_PRICING_CACHE if _MODEL_PRICING_CACHE is None: @@ -78,118 +84,128 @@ def get_model_pricing(model_id: str) -> dict: _MODEL_PRICING_CACHE = {} except Exception: _MODEL_PRICING_CACHE = {} - + # Check if exact match exists, if not, try partial matches (e.g., if version suffixes differ) if _MODEL_PRICING_CACHE and model_id not in _MODEL_PRICING_CACHE: for k, v in _MODEL_PRICING_CACHE.items(): if model_id in k or k in model_id: return v - + return _MODEL_PRICING_CACHE.get(model_id, {}) + def prewarm_ollama_models(configs): """ - Sends a dummy request to the configured local Ollama API endpoints via a background thread + Sends a dummy request to the configured local Ollama API endpoints via a background thread to force the models to load into VRAM during bot startup, minimizing initial connection latency and avoiding timeouts downstream. """ args = configs.args - + def _warmup(): - import threading models_to_warm = set() - + # Collect unique local models for attr, url_attr in [ ("ai_telepathic_model", "ai_telepathic_url"), ("ai_fallback_model", "ai_fallback_url"), ("ai_condenser_model", "ai_condenser_url"), - ("ai_model", "ai_model_url") + ("ai_model", "ai_model_url"), ]: url = getattr(args, url_attr, "") model = getattr(args, attr, "") if model and url and ("localhost" in url or "127.0.0.1" in url): models_to_warm.add((url, model)) - + for url, model in models_to_warm: - logger.info(f"🔥 [VRAM Pre-Warm] Instructing local Ollama engine to load {model} into memory in the background...") + logger.info( + f"🔥 [VRAM Pre-Warm] Instructing local Ollama engine to load {model} into memory in the background..." + ) try: # Fire an ultra-short generation to force it into VRAM requests.post( - url, - json={"model": model, "prompt": "Hi", "stream": False, "options": {"num_predict": 1}}, - timeout=120 + url, + json={"model": model, "prompt": "Hi", "stream": False, "options": {"num_predict": 1}}, + timeout=120, ) except Exception: pass - + if hasattr(args, "ai_telepathic_model"): import threading + threading.Thread(target=_warmup, daemon=True).start() + def unload_ollama_models(configs): """ - Sends keep_alive: 0 to all configured local Ollama API endpoints via a background thread + Sends keep_alive: 0 to all configured local Ollama API endpoints via a background thread to force the models to unload from VRAM during bot shutdown. """ args = configs.args - + def _unload(): - import threading models_to_unload = set() - + # Collect unique local models for attr, url_attr in [ ("ai_telepathic_model", "ai_telepathic_url"), ("ai_fallback_model", "ai_fallback_url"), ("ai_condenser_model", "ai_condenser_url"), - ("ai_model", "ai_model_url") + ("ai_model", "ai_model_url"), ]: url = getattr(args, url_attr, "") model = getattr(args, attr, "") if model and url and ("localhost" in url or "127.0.0.1" in url): models_to_unload.add((url, model)) - + for url, model in models_to_unload: logger.info(f"❄️ [VRAM Cleanup] Instructing local Ollama engine to unload {model} from memory...") try: # Fire keep_alive: 0 to unload it from VRAM - requests.post( - url, - json={"model": model, "keep_alive": 0}, - timeout=5 - ) + requests.post(url, json={"model": model, "keep_alive": 0}, timeout=5) except Exception as e: logger.debug(f"Failed to unload {model}: {e}") - + if hasattr(args, "ai_telepathic_model"): import threading + threading.Thread(target=_unload, daemon=True).start() + def log_openrouter_burn(): """Fetches and logs the current OpenRouter API key usage (money burned) ONLY if OpenRouter is actively used.""" key = os.environ.get("OPENROUTER_API_KEY") if not key: return - + try: from GramAddict.core.config import Config + args = Config().args uses_openrouter = False - + # Check all possible model/url endpoints for 'openrouter' - for attr in ["ai_model", "ai_model_url", "ai_telepathic_model", "ai_telepathic_url", - "ai_fallback_model", "ai_fallback_url", "ai_condenser_model", "ai_condenser_url"]: + for attr in [ + "ai_model", + "ai_model_url", + "ai_telepathic_model", + "ai_telepathic_url", + "ai_fallback_model", + "ai_fallback_url", + "ai_condenser_model", + "ai_condenser_url", + ]: val = getattr(args, attr, "") if val and "openrouter" in str(val).lower(): uses_openrouter = True break - + if not uses_openrouter: return except Exception: pass - + try: r = requests.get("https://openrouter.ai/api/v1/auth/key", headers={"Authorization": f"Bearer {key}"}, timeout=5) if r.status_code == 200: @@ -197,11 +213,16 @@ def log_openrouter_burn(): total_spent = data.get("usage", 0.0) daily_spent = data.get("usage_daily", 0.0) limit = data.get("limit") - - logger.info(f"🔥 [OpenRouter Burn Rate] Daily: ${daily_spent:.4f} | Total: ${total_spent:.4f}" + (f" | Limit: ${limit}" if limit else ""), extra={"color": "\x1b[38;5;208m\x1b[1m"}) + + logger.info( + f"🔥 [OpenRouter Burn Rate] Daily: ${daily_spent:.4f} | Total: ${total_spent:.4f}" + + (f" | Limit: ${limit}" if limit else ""), + extra={"color": "\x1b[38;5;208m\x1b[1m"}, + ) except Exception as e: logger.debug(f"Could not fetch OpenRouter burn rate: {e}") + def query_llm( url: str, model: str, @@ -213,16 +234,16 @@ def query_llm( fallback_model: Optional[str] = None, fallback_url: Optional[str] = None, temperature: Optional[float] = None, - max_tokens: Optional[int] = None + max_tokens: Optional[int] = None, ) -> Optional[dict]: """ Unified LLM API Caller with configurable fallback. """ openrouter_key = os.environ.get("OPENROUTER_API_KEY") - + # URL-based provider detection (not model-name based — works for any model) is_openai_compat = "/v1/chat/completions" in url or "openrouter.ai" in url.lower() or "openai.com" in url.lower() - + # If using a cloud model but a local URL was passed, fix it if not is_openai_compat and ("openrouter" in model.lower() or "/" in model): # Model looks like "org/model-name" which is OpenRouter format @@ -230,54 +251,43 @@ def query_llm( url = "https://openrouter.ai/api/v1/chat/completions" headers = {"Content-Type": "application/json"} - + if is_openai_compat: if openrouter_key: headers["Authorization"] = f"Bearer {openrouter_key}" - + messages = [] if system: messages.append({"role": "system", "content": system}) - + user_content = [] if prompt: user_content.append({"type": "text", "text": prompt}) - + if images_b64: for img in images_b64: - user_content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img}"} - }) - + user_content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img}"}}) + messages.append({"role": "user", "content": user_content if len(user_content) > 1 else prompt}) - - req_data = { - "model": model, - "messages": messages, - "stream": False - } + + req_data = {"model": model, "messages": messages, "stream": False} if format_json: req_data["response_format"] = {"type": "json_object"} if temperature is not None: req_data["temperature"] = temperature if max_tokens is not None: req_data["max_tokens"] = max_tokens - + else: # Ollama /generate API - req_data = { - "model": model, - "prompt": prompt, - "stream": False - } + req_data = {"model": model, "prompt": prompt, "stream": False} if system: req_data["system"] = system if images_b64: req_data["images"] = images_b64 if format_json: req_data["format"] = "json" - + # Ollama passes configs inside 'options' if temperature is not None or max_tokens is not None: req_data["options"] = {} @@ -290,12 +300,12 @@ def query_llm( response = requests.post(url, json=req_data, headers=headers, timeout=timeout) response.raise_for_status() resp_json = response.json() - + # Normalize response payload so callers don't have to distinguish if is_openai_compat: # OpenRouter returns choices[0].message.content content = resp_json.get("choices", [{}])[0].get("message", {}).get("content", "") - + usage = resp_json.get("usage", {}) if usage: cost_str = "" @@ -306,26 +316,29 @@ def query_llm( pricing = get_model_pricing(model) if pricing: try: - p_cost = float(pricing.get("prompt", 0)) * usage.get('prompt_tokens', 0) - c_cost = float(pricing.get("completion", 0)) * usage.get('completion_tokens', 0) + p_cost = float(pricing.get("prompt", 0)) * usage.get("prompt_tokens", 0) + c_cost = float(pricing.get("completion", 0)) * usage.get("completion_tokens", 0) calc_cost = p_cost + c_cost if calc_cost > 0: cost_str = f" | 💸 Cost: ${calc_cost:.6f}" except Exception: pass - - p_tokens = usage.get('prompt_tokens', 0) - c_tokens = usage.get('completion_tokens', 0) - t_tokens = usage.get('total_tokens', 0) - + + p_tokens = usage.get("prompt_tokens", 0) + c_tokens = usage.get("completion_tokens", 0) + t_tokens = usage.get("total_tokens", 0) + # Make it stand out! - logger.info(f"🪙 [LLM Burn] {model} -> In: {p_tokens} | Out: {c_tokens} | Total: {t_tokens}{cost_str}", extra={"color": "\x1b[38;5;208m\x1b[1m"}) - + logger.info( + f"🪙 [LLM Burn] {model} -> In: {p_tokens} | Out: {c_tokens} | Total: {t_tokens}{cost_str}", + extra={"color": "\x1b[38;5;208m\x1b[1m"}, + ) + # Validation: if JSON was expected, try to extract it if format_json: extracted = extract_json(content) if not extracted: - raise ValueError(f"OpenRouter returned non-JSON content when JSON was expected: {content[:100]}...") + raise ValueError(f"OpenRouter returned non-JSON content when JSON was expected: {content[:100]}...") content = extracted return {"response": content} @@ -337,31 +350,34 @@ def query_llm( if not extracted: # Log more context if JSON extraction fails logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...") - raise ValueError(f"Ollama returned non-JSON content when JSON was expected.") + raise ValueError("Ollama returned non-JSON content when JSON was expected.") resp_json["response"] = extracted return resp_json + except requests.exceptions.ConnectionError: + logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?") except Exception as e: logger.error(f"LLM Provider Error with {model}: {e}") - + # Prevent infinite fallback loops if getattr(query_llm, "_is_fallback", False): return None - + # Decide on fallback model/url f_model = fallback_model f_url = fallback_url - + # Read fallback config from args if available if not f_model or not f_url: from GramAddict.core.config import Config + try: args = Config().args f_model = f_model or getattr(args, "ai_fallback_model", None) f_url = f_url or getattr(args, "ai_fallback_url", None) except Exception: pass - + # Last resort defaults if not f_model or not f_url: if is_openai_compat: @@ -388,12 +404,13 @@ def query_llm( format_json=format_json, timeout=timeout, temperature=temperature, - max_tokens=max_tokens + max_tokens=max_tokens, ) finally: query_llm._is_fallback = False return None + def query_telepathic_llm( model: str, url: str, @@ -401,7 +418,7 @@ def query_telepathic_llm( user_prompt: str, temperature: float = 0.0, use_local_edge: bool = False, - images_b64: Optional[List[str]] = None + images_b64: Optional[List[str]] = None, ) -> str: """ Routes UI Telepathic requests purely based on textual interpretation of the screen's XML nodes. @@ -415,10 +432,13 @@ def query_telepathic_llm( if use_local_edge: is_already_local = "localhost" in url or "127.0.0.1" in url if is_already_local: - logger.debug(f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing.") + logger.debug( + f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing." + ) else: logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).") from GramAddict.core.config import Config + try: args = Config().args target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate") @@ -426,10 +446,10 @@ def query_telepathic_llm( except Exception: target_url = "http://localhost:11434/api/generate" target_model = "llama3.2:1b" - + is_local = "localhost" in target_url or "127.0.0.1" in target_url calc_timeout = 180 if is_local else 45 - + ans = query_llm( url=target_url, model=target_model, @@ -439,7 +459,7 @@ def query_telepathic_llm( format_json=True, timeout=calc_timeout, # Navigation VLM must fail fast for Cloud, but wait for Local VRAM loads temperature=temperature, - max_tokens=150 # Hard stop to prevent VLM from endlessly hallucinating UI elements + max_tokens=150, # Hard stop to prevent VLM from endlessly hallucinating UI elements ) if ans and "response" in ans: return ans["response"] diff --git a/GramAddict/core/navigation/__init__.py b/GramAddict/core/navigation/__init__.py new file mode 100644 index 0000000..a4ded28 --- /dev/null +++ b/GramAddict/core/navigation/__init__.py @@ -0,0 +1 @@ +# Navigation domain package diff --git a/GramAddict/core/navigation/knowledge.py b/GramAddict/core/navigation/knowledge.py new file mode 100644 index 0000000..6b0f389 --- /dev/null +++ b/GramAddict/core/navigation/knowledge.py @@ -0,0 +1,214 @@ +import logging +import time +from typing import List, Optional + +from GramAddict.core.perception.screen_identity import ScreenType +from GramAddict.core.qdrant_memory import QdrantBase + +logger = logging.getLogger(__name__) + + +class NavigationKnowledge: + """ + Manages the bot's learned understanding of the Instagram UI. + Discovered dynamically through exploration and success. + """ + + def __init__(self, username: str): + self.username = username + try: + self._db = QdrantBase("navigation_knowledge", vector_size=768) + except Exception: + self._db = None + + # In-memory cache for rapidly avoiding traps during exploration + # In-memory cache for rapidly avoiding traps during exploration + self._learned_screen_mappings = {} + self._learned_traps = set() + + def wipe(self): + """Wipe all learned knowledge from Qdrant.""" + if self._db and self._db.is_connected: + try: + self._db.wipe_collection() + except Exception as e: + logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}") + + def update_username(self, username: str): + """Update username and reconnect DB if needed.""" + if self.username != username: + self.username = username + try: + self._db = QdrantBase("navigation_knowledge", vector_size=768) + except Exception: + self._db = None + + def get_requirements(self, goal: str) -> List[ScreenType]: + """Get required screens for a goal. Returns known requirements or empty list.""" + if not self._db or not self._db.is_connected: + return [] + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.scroll( + collection_name=self._db.collection_name, + scroll_filter=Filter(must=[FieldCondition(key="goal", match=MatchValue(value=goal))]), + limit=1, + )[0] + if results: + screen_name = results[0].payload.get("required_screen") + logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}") + if screen_name: + return [ScreenType[screen_name]] + except Exception as e: + logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}") + return [] + + def learn_goal_requirement(self, goal: str, screen_type: ScreenType): + """Learn that achieving 'goal' lands us on 'screen_type'.""" + if not self._db or not self._db.is_connected: + logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected") + return + + seed = f"req_{goal}" + vec = self._db._get_embedding(f"goal_requirement: {goal}") + payload = {"goal": goal, "required_screen": screen_type.name, "timestamp": time.time()} + self._db.upsert_point(seed, payload, vector=vec) + logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}' → {screen_type.name}") + + def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]: + """Find which action leads to this screen.""" + for action, screen in self._learned_screen_mappings.items(): + if screen == target_screen: + return action + + if not self._db or not self._db.is_connected: + return None + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.scroll( + collection_name=self._db.collection_name, + scroll_filter=Filter( + must=[FieldCondition(key="result_screen", match=MatchValue(value=target_screen.name))] + ), + limit=1, + )[0] + if results: + return results[0].payload.get("action") + except Exception: + pass + return None + + def get_screen_for_action(self, action: str) -> Optional[ScreenType]: + """Find where this action leads to to avoid looping traps.""" + if action in self._learned_screen_mappings: + return self._learned_screen_mappings[action] + + if not self._db or not self._db.is_connected: + return None + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.scroll( + collection_name=self._db.collection_name, + scroll_filter=Filter(must=[FieldCondition(key="action", match=MatchValue(value=action))]), + limit=1, + )[0] + if results: + screen_name = results[0].payload.get("result_screen") + if screen_name: + return ScreenType[screen_name] + except Exception: + pass + return None + + def learn_screen_mapping(self, action: str, result_screen: ScreenType): + """Learn that taking 'action' leads to 'result_screen'.""" + if not self._db or not self._db.is_connected: + return + + seed = f"map_{action}" + vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}") + payload = {"action": action, "result_screen": result_screen.name, "timestamp": time.time()} + + self._learned_screen_mappings[action] = result_screen + + self._db.upsert_point(seed, payload, vector=vec) + logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}' → {result_screen.name}") + + def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]: + """Find where this tab leads to to avoid looping traps.""" + if tab_id in self._learned_screen_mappings: + return self._learned_screen_mappings[tab_id] + + if not self._db or not self._db.is_connected: + return None + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.scroll( + collection_name=self._db.collection_name, + scroll_filter=Filter(must=[FieldCondition(key="tab_id", match=MatchValue(value=tab_id))]), + limit=1, + )[0] + if results: + s_name = results[0].payload.get("result_screen") + if s_name: + return ScreenType[s_name] + except Exception: + pass + return None + + def learn_trap(self, screen_type: ScreenType, action: str, trap_reason: str = "softlock"): + """Aversively learn that an action on a screen is dangerous/useless.""" + trap_key = f"{screen_type.name}_{action}" + self._learned_traps.add(trap_key) + + if not self._db or not self._db.is_connected: + return + + seed = f"trap_{trap_key}" + # Aversive vector is completely orthogonal to normal goals to prevent retrieval overlap + vec = self._db._get_embedding(f"trap_avoidance: {trap_key} {trap_reason}") + payload = { + "trap_screen": screen_type.name, + "trap_action": action, + "trap_reason": trap_reason, + "timestamp": time.time(), + } + self._db.upsert_point(seed, payload, vector=vec) + logger.error(f"💀 [Aversive Learning] BURNED action '{action}' on {screen_type.name} due to: {trap_reason}") + + def is_trap(self, screen_type: ScreenType, action: str) -> bool: + """Check if an action on this screen is a known trap.""" + trap_key = f"{screen_type.name}_{action}" + if trap_key in self._learned_traps: + return True + + if not self._db or not self._db.is_connected: + return False + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.scroll( + collection_name=self._db.collection_name, + scroll_filter=Filter( + must=[ + FieldCondition(key="trap_screen", match=MatchValue(value=screen_type.name)), + FieldCondition(key="trap_action", match=MatchValue(value=action)), + ] + ), + limit=1, + )[0] + if results: + self._learned_traps.add(trap_key) + return True + except Exception: + pass + return False diff --git a/GramAddict/core/navigation/path_memory.py b/GramAddict/core/navigation/path_memory.py new file mode 100644 index 0000000..e737dc8 --- /dev/null +++ b/GramAddict/core/navigation/path_memory.py @@ -0,0 +1,117 @@ +import logging +import time +from typing import Dict, List, Optional + +from GramAddict.core.qdrant_memory import QdrantBase + +logger = logging.getLogger(__name__) + + +class PathMemory: + """ + Qdrant-backed memory for successful navigation paths. + + Stores: goal → [step1, step2, ...] → success + Enables instant recall for known goals. + """ + + def __init__(self, username: str = ""): + self.username = username + try: + suffix = f"_{username}" if username else "" + self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768) + except Exception: + self._db = None + + def wipe(self): + """Wipe all learned navigation paths from Qdrant.""" + if self._db and self._db.is_connected: + try: + self._db.wipe_collection() + except Exception as e: + logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}") + + def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]: + """ + Recall a previously successful path for this goal from this screen type. + Returns list of steps or None. + """ + if not self._db or not self._db.is_connected: + return None + + query = f"goal: {goal} | from: {current_screen_type}" + vec = self._db._get_embedding(query) + if not vec: + return None + + try: + from qdrant_client.models import FieldCondition, Filter, MatchValue + + results = self._db.client.query_points( + collection_name=self._db.collection_name, + query=vec, + query_filter=Filter( + must=[FieldCondition(key="start_screen", match=MatchValue(value=current_screen_type))] + ), + limit=3, + score_threshold=0.85, + ).points + + for r in results: + p = r.payload + if p.get("success") and p.get("steps"): + logger.info( + f"🧠 [GOAP Recall] Found path for '{goal}': " + f"{len(p['steps'])} steps (confidence: {p.get('confidence', 0):.2f})" + ) + return p["steps"] + + return None + except Exception as e: + logger.debug(f"GOAP recall error: {e}") + return None + + def learn_path(self, goal: str, start_screen: str, steps: List[Dict], success: bool): + """Store a navigation path in Qdrant.""" + if not self._db or not self._db.is_connected: + return + + query = f"goal: {goal} | from: {start_screen}" + vec = self._db._get_embedding(query) + if not vec: + return + + seed = f"{goal}|{start_screen}" + payload = { + "goal": goal, + "start_screen": start_screen, + "steps": steps, + "step_count": len(steps), + "success": success, + "confidence": 0.85 if success else 0.0, + "timestamp": time.time(), + } + + outcome = "✅" if success else "❌" + self._db.upsert_point( + seed, + payload, + vector=vec, + log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}", + ) + + def forget_path(self, goal: str, start_screen: str): + """Remove a cached path to force re-discovery.""" + if not self._db or not self._db.is_connected: + return + + seed = f"{goal}|{start_screen}" + try: + from qdrant_client import models + + point_id = self._db._get_id(seed) + self._db.client.delete( + collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id]) + ) + except Exception as e: + logger.debug(f"Failed to forget path: {e}") diff --git a/GramAddict/core/navigation/planner.py b/GramAddict/core/navigation/planner.py new file mode 100644 index 0000000..dad1687 --- /dev/null +++ b/GramAddict/core/navigation/planner.py @@ -0,0 +1,171 @@ +import logging +from typing import Any, Dict, List, Optional + +from GramAddict.core.navigation.knowledge import NavigationKnowledge +from GramAddict.core.perception.screen_identity import ScreenType + +logger = logging.getLogger(__name__) + + +class GoalPlanner: + """ + Given a goal and current screen state, plans the next action. + + Uses Dynamic Discovery to navigate without hardcoded maps. + """ + + def __init__(self, username: str): + self.knowledge = NavigationKnowledge(username) + + def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]: + """Plans the NEXT single action to take toward the goal.""" + screen_type = screen["screen_type"] + available = screen.get("available_actions", []) + context = screen.get("context", {}) + goal_lower = goal.lower() + + # ── 1. Check if goal is ALREADY achieved ── + if self._is_goal_achieved(goal_lower, screen_type, context): + logger.info(f"🎯 [GOAP] Goal '{goal}' already achieved on {screen_type.value}!") + return None + + # (Phase 5: legacy _plan_goal_action static heuristics purged, + # all intents fall through to VLM-driven Discovery in _plan_navigation) + + # ── 3. Am I on the right screen? If not, navigate there ── + selected_tab = screen.get("selected_tab") + nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions) + if nav_action: + return nav_action + + # Final fallback: back-track, UNLESS back-tracking is a known trap on this screen! + if not self.knowledge.is_trap(screen_type, "press back"): + return "press back" + + # We are trapped! Can't go forward, can't go back! + logger.error(f"💀 [GOAP] Completely trapped on {screen_type.name}. Forcing Instagram restart.") + return "force start instagram" + + def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool: + """Check if the goal is already satisfied. Delegates to ScreenTopology SSOT.""" + from GramAddict.core.screen_topology import ScreenTopology + + # Interaction goals (context-specific, not navigation) + if "like" in goal and context.get("is_liked") is True: + return True + if "view profile" in goal and screen_type in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE): + return True + + # Navigation goals — delegate to SSOT + target = ScreenTopology.goal_to_target_screen(goal) + if target and screen_type == target: + return True + + return False + + def _plan_navigation( + self, + goal: str, + screen_type: ScreenType, + available: List[str], + selected_tab: Optional[str] = None, + explored_nav_actions: set = None, + ) -> Optional[str]: + """If we're on the wrong screen, figure out how to navigate. + + Strategy (priority order): + 1. HD Map (ScreenTopology BFS) — deterministic, pre-computed routes + 2. Learned Knowledge (Qdrant) — dynamic discovery from past sessions + 3. Autonomous Discovery — linguistic matching + VLM intent + """ + from GramAddict.core.screen_topology import ScreenTopology + + # 0. Aversive Filter: Remove known traps from available actions + safe_available = [] + for action in available: + if not self.knowledge.is_trap(screen_type, action): + safe_available.append(action) + else: + logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'") + available = safe_available + + # ── 1. HD Map Routing (Primary Strategy) ── + target_screen = ScreenTopology.goal_to_target_screen(goal) + if target_screen and target_screen != screen_type: + route = ScreenTopology.find_route(screen_type, target_screen) + if route: + next_action, next_screen = route[0] + # Verify action isn't explored/trapped + if next_action not in (explored_nav_actions or set()): + if not self.knowledge.is_trap(screen_type, next_action): + route_desc = " → ".join(s.name for _, s in route) + logger.info( + f"🗺️ [HD Map] Route: {screen_type.name} → {route_desc}. " f"Next action: '{next_action}'" + ) + return next_action + else: + logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.") + else: + logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.") + + # ── 2. Learned Knowledge (Qdrant) ── + required_screens = self.knowledge.get_requirements(goal) + + # ── 3. Autonomous Discovery (Blank Start fallback) ── + if not required_screens: + logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.") + + # Return raw intent for TelepathicEngine discovery (VLM) + if explored_nav_actions and goal in explored_nav_actions: + logger.info( + f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking." + ) + return None # Don't return goal again — force fallback to press back + else: + return goal + + # 4. If we're already on an acceptable screen, no navigation needed + if screen_type in required_screens: + return None + + # 5. Find the action we need to take (from learned knowledge or HD map) + for target_screen in required_screens: + # Try HD Map first! + route = ScreenTopology.find_route(screen_type, target_screen) + if route: + next_action, next_screen = route[0] + if next_action not in (explored_nav_actions or set()): + if not self.knowledge.is_trap(screen_type, next_action): + logger.info(f"🧭 [Nav HD Map] Routing to required {target_screen.name} via '{next_action}'") + return next_action + + known_action = self.knowledge.get_action_for_screen(target_screen) + + if not known_action: + logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...") + + screen_friendly_name = target_screen.name.replace("_", " ").lower() + goal_words = [w.rstrip("s") for w in screen_friendly_name.split() if len(w) > 3] + + for action in available: + if any(w in action.lower() for w in goal_words): + known_target = self.knowledge.get_screen_for_action(action) + if known_target and known_target != target_screen: + continue + + logger.info( + f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'" + ) + return action + + return f"navigate to {screen_friendly_name}" + else: + if known_action in available: + logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'") + return known_action + + # If no targeted navigation works, try going back first + if "press back" in available: + return "press back" + + return None diff --git a/GramAddict/core/perception/action_memory.py b/GramAddict/core/perception/action_memory.py new file mode 100644 index 0000000..94b513d --- /dev/null +++ b/GramAddict/core/perception/action_memory.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, Dict, Optional + +from GramAddict.core.perception.spatial_parser import SpatialNode + +logger = logging.getLogger(__name__) + + +class ActionMemory: + """ + Handles the caching, tracking, and negative reinforcement (unlearning) of UI interactions. + Decouples the memory layer from the core parsing engine. + """ + + def __init__(self, ui_memory=None): + # We optionally inject UIMemoryDB to decouple tests + if ui_memory is None: + from GramAddict.core.qdrant_memory import UIMemoryDB + + self.ui_memory = UIMemoryDB() + else: + self.ui_memory = ui_memory + + self._last_click_context: Optional[Dict[str, Any]] = None + + def track_click(self, intent: str, node: SpatialNode, xml_context: str = ""): + """Stores the context of a click before it's actually performed.""" + semantic_string = f"text: '{node.text}', desc: '{node.content_desc}', id: '{node.resource_id}'" + + self._last_click_context = { + "intent": intent, + "node_dict": node.to_dict(), + "semantic_string": semantic_string, + "xml_context": xml_context, + } + logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}") + + def confirm_click(self, intent: str = None): + """Positive Reinforcement: Confirms the last click was successful.""" + ctx = self._last_click_context + if not ctx: + return + + if intent and ctx["intent"] != intent: + return + + logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.") + + # Store or boost in Qdrant + try: + # Check if it exists first + existing = self.ui_memory.retrieve_memory(ctx["intent"], ctx["xml_context"]) + if existing: + self.ui_memory.boost_confidence(ctx["intent"], ctx["xml_context"]) + else: + self.ui_memory.store_memory(ctx["intent"], ctx["xml_context"], ctx["node_dict"]) + except Exception as e: + logger.warning(f"Failed to confirm click in Qdrant: {e}") + + self._last_click_context = None + + def reject_click(self, intent: str = None): + """Negative Reinforcement: Penalizes a failed click (Unlearning).""" + ctx = self._last_click_context + if not ctx: + return + + if intent and ctx["intent"] != intent: + return + + logger.warning(f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.") + + try: + self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"]) + except Exception as e: + logger.warning(f"Failed to decay confidence in Qdrant: {e}") + + self._last_click_context = None + + def verify_success(self, intent: str, pre_click_xml: str, post_click_xml: str) -> Optional[bool]: + """ + Structural verification: Did the UI actually change after the click? + """ + # Specific check for explore grid + if "first image in explore grid" in intent or "grid item" in intent: + if "row_feed_photo_imageview" in post_click_xml or "row_feed_button_like" in post_click_xml: + return True + if "explore_action_bar" in post_click_xml and "row_feed_button_like" not in post_click_xml: + return None # Still on grid, inconclusive + + if abs(len(pre_click_xml) - len(post_click_xml)) > 50: + logger.debug(f"🧠 [ActionMemory] Structural change detected for '{intent}'. Verification PASS.") + return True + + logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.") + return False diff --git a/GramAddict/core/perception/feed_analysis.py b/GramAddict/core/perception/feed_analysis.py index 428062f..de208cd 100644 --- a/GramAddict/core/perception/feed_analysis.py +++ b/GramAddict/core/perception/feed_analysis.py @@ -1,7 +1,7 @@ """ Perception — Feed Content Analysis. -Structural analysis of the feed: detecting markers, carousels, +Structural analysis of the feed: detecting markers, carousels, extracting post content. Zero-AI, pure structural parsing. Extracted from bot_flow.py to enable isolated testing. @@ -27,14 +27,14 @@ FEED_MARKERS = [ "clips_linear_layout_container", "zoomable_view_container", "feed_action_row", - "carousel_viewpager" + "carousel_viewpager", ] # ── Carousel Detection ── CAROUSEL_INDICATORS = [ "com.instagram.android:id/carousel_page_indicator", "com.instagram.android:id/carousel_media_group", - "com.instagram.android:id/carousel_viewpager" + "com.instagram.android:id/carousel_viewpager", ] @@ -50,26 +50,29 @@ def extract_post_content(context_xml: str) -> dict: """ Extracts meaningful content data from the current feed post's XML. This is the BOT'S EYES — what it actually "sees" about each post. - + Returns: {'username': str, 'description': str, 'caption': str} """ result = {"username": "", "description": "", "caption": ""} - + try: from GramAddict.core.telepathic_engine import TelepathicEngine + telepath = TelepathicEngine.get_instance() - + # 1. Learn/extract post author dynamically author_node = telepath.find_best_node(context_xml, "post author username header", min_confidence=0.75) - if author_node and author_node.get("original_attribs", {}).get("text"): + + # 🛡️ Anti-Hallucination Guard: The author header is always near the top. Ignore names in the comment section. + if author_node and author_node.get("y", 0) < 1000 and author_node.get("original_attribs", {}).get("text"): result["username"] = author_node["original_attribs"]["text"].strip() - + # 2. Learn/extract post media description dynamically media_node = telepath.find_best_node(context_xml, "post media content", min_confidence=0.35) if media_node and media_node.get("original_attribs", {}).get("desc"): result["description"] = media_node["original_attribs"]["desc"].strip() - + # 3. Visible caption text (heuristic fallback if node isn't explicitly found) # Search all nodes for text that contains the username to find the caption body root = ET.fromstring(context_xml) @@ -78,13 +81,51 @@ def extract_post_content(context_xml: str) -> dict: if result["username"] and len(text) > 20 and result["username"] in text: result["caption"] = text break - + except Exception as e: logger.warning(f"Error extracting post content autonomously: {e}") - + return result +def _parse_number_from_text(text: str) -> int: + """Extracts numeric value from strings like '1,234 likes', '1.5M views', 'Gefällt 12.345 Mal'.""" + text = text.lower() + + # Clean up purely thousands separators but keep decimals + # If there is a 'm' or 'k', a period is usually a decimal (e.g. 1.5m). + # If no 'm' or 'k', a period might be a German thousands separator (12.345). + # We will let the regex handle decimals. + + # Remove commas (usually thousands separator in English) + text = text.replace(",", "") + + # Find all numbers, potentially with k or m + matches = re.findall(r"(\d+(?:\.\d+)?)\s*([km])?", text) + if not matches: + return 0 + + best_val = 0 + for num_str, multiplier in matches: + val = float(num_str) + if multiplier == "k": + val *= 1000 + elif multiplier == "m": + val *= 1000000 + else: + # If no multiplier, a period in num_str might be a German thousands separator + if "." in num_str and val < 1000: + # E.g. '12.345' became 12.345. Since no multiplier, it's actually 12345. + # Heuristic: If it has 3 decimal places, it's a thousands separator. + parts = num_str.split(".") + if len(parts[1]) == 3: + val = float(num_str.replace(".", "")) + + best_val = max(best_val, int(val)) + + return best_val + + def has_feed_markers(xml_dump: str) -> bool: """Quick check: does this XML contain any feed presence markers?""" return any(marker in xml_dump for marker in FEED_MARKERS) diff --git a/GramAddict/core/perception/screen_identity.py b/GramAddict/core/perception/screen_identity.py new file mode 100644 index 0000000..9eb8f1e --- /dev/null +++ b/GramAddict/core/perception/screen_identity.py @@ -0,0 +1,349 @@ +import hashlib +import logging +import re +import xml.etree.ElementTree as ET +from enum import Enum +from typing import Any, Dict + +logger = logging.getLogger(__name__) + + +class ScreenType(Enum): + HOME_FEED = "home_feed" + EXPLORE_GRID = "explore_grid" + REELS_FEED = "reels_feed" + OWN_PROFILE = "own_profile" + OTHER_PROFILE = "other_profile" + POST_DETAIL = "post_detail" + STORY_VIEW = "story_view" + DM_INBOX = "dm_inbox" + DM_THREAD = "dm_thread" + SEARCH_RESULTS = "search_results" + FOLLOW_LIST = "follow_list" + COMMENTS = "comments" + MODAL = "modal" + FOREIGN_APP = "foreign_app" + UNKNOWN = "unknown" + + +class ScreenIdentity: + """ + Understands what screen the bot is on by analyzing the XML dump. + NO hardcoded states — purely structural analysis. + + This is the bot's EYES. It answers: "What do I see right now?" + """ + + def __init__(self, bot_username: str): + self.bot_username = bot_username.lower() + try: + from GramAddict.core.qdrant_memory import ScreenMemoryDB + + self.screen_memory = ScreenMemoryDB() + except ImportError: + self.screen_memory = None + + def identify(self, xml_dump: str) -> Dict[str, Any]: + """ + Analyzes an XML dump and returns a complete screen description. + + Returns: + { + 'screen_type': ScreenType, + 'available_actions': ['tap like button', 'tap explore tab', ...], + 'selected_tab': 'feed_tab' | 'search_tab' | ..., + 'context': {'username': '...', 'post_count': '...', ...} + } + """ + if not xml_dump or not isinstance(xml_dump, str): + return self._empty_screen() + + try: + clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip() + root = ET.fromstring(clean) + except Exception: + return self._empty_screen() + + # Extract structural signals + packages = set() + resource_ids = set() + content_descs = [] + texts = [] + selected_tab = None + clickable_elements = [] + + app_id = "com.instagram.android" + + for elem in root.iter("node"): + pkg = elem.get("package", "") + if pkg: + packages.add(pkg) + + rid = elem.get("resource-id", "").strip() + text = elem.get("text", "").strip() + desc = elem.get("content-desc", "").strip() + clickable = elem.get("clickable", "false") == "true" + selected = elem.get("selected", "false") == "true" + bounds = elem.get("bounds", "") + + if rid: + # Normalize: "com.instagram.android:id/feed_tab" → "feed_tab" + short_id = rid.split("/")[-1] if "/" in rid else rid + resource_ids.add(short_id) + + # Track which tab is selected + if selected and short_id in ("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab"): + selected_tab = short_id + + if text: + texts.append(text) + if desc: + content_descs.append(desc) + + if clickable and bounds: + match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds) + if match: + left, t, r, b = map(int, match.groups()) + cx, cy = (left + r) // 2, (t + b) // 2 + clickable_elements.append( + { + "text": text, + "desc": desc, + "id": rid.split("/")[-1] if "/" in rid else rid, + "x": cx, + "y": cy, + "bounds": bounds, + } + ) + + # ── Foreign app check ── + if app_id not in packages: + return { + "screen_type": ScreenType.FOREIGN_APP, + "available_actions": ["press back", "force start instagram"], + "selected_tab": None, + "context": {"packages": list(packages)}, + "signature": self._compute_signature(resource_ids, content_descs, texts), + } + + desc_lower = " ".join(content_descs).lower() + text_lower = " ".join(texts).lower() + ids_str = " ".join(resource_ids).lower() + + signature = self._compute_signature(resource_ids, content_descs, texts) + + # ── Identify screen type from structural signals ── + screen_type = self._classify_screen( + resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature + ) + + # ── Extract available actions from clickable elements ── + available_actions = self._extract_available_actions( + clickable_elements, resource_ids, content_descs, texts, screen_type + ) + + # ── Extract context ── + context = self._extract_context(content_descs, texts, resource_ids, screen_type) + + return { + "screen_type": screen_type, + "available_actions": available_actions, + "selected_tab": selected_tab, + "context": context, + "signature": signature, + } + + def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None): + """Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states.""" + + # Priority 0: Content-creation overlays that block ALL navigation. + # These full-screen Instagram UIs have no navigation tabs and trap the bot. + # Structural detection is O(1), zero LLM calls, and cannot be fooled. + creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera") + if any(marker in ids_str for marker in creation_flow_markers): + logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL") + return ScreenType.MODAL + + # Priority 1: Check Qdrant Semantic Cache + if signature and self.screen_memory and self.screen_memory.is_connected: + cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92) + if cached_type_str: + try: + return ScreenType[cached_type_str] + except KeyError: + pass + + # Priority 2: Structural Heuristics (Instant, for core tabs) + if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids: + return ScreenType.FOLLOW_LIST + + if "profile_header_container" in ids: + return ScreenType.OTHER_PROFILE + + # Reels structural markers — present even when Instagram hides the tab bar + # in full-screen Reels viewing. Without this, selected_tab=None → UNKNOWN. + REELS_MARKERS = ("clips_viewer_container", "root_clips_layout", "clips_linear_layout_container") + if any(marker in ids for marker in REELS_MARKERS): + return ScreenType.REELS_FEED + + # DM thread detection — structural markers present inside DM conversations + if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids: + return ScreenType.DM_THREAD + + if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab: + return ScreenType.POST_DETAIL + + if selected_tab == "feed_tab": + return ScreenType.HOME_FEED + if selected_tab == "clips_tab": + return ScreenType.REELS_FEED + if selected_tab == "search_tab": + return ScreenType.EXPLORE_GRID + if selected_tab == "profile_tab": + return ScreenType.OWN_PROFILE + if selected_tab == "direct_tab": + return ScreenType.DM_INBOX + if "message_input" in ids: + return ScreenType.DM_INBOX # Fallback for DM thread as inbox + + # Priority 3: Semantic VLM Classification Fallback + from GramAddict.core.config import Config + from GramAddict.core.llm_provider import query_llm + + cfg = Config() + url = ( + getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat") + if hasattr(cfg, "args") + else "http://localhost:11434/api/chat" + ) + model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3" + + layout_context = ( + f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n" + ) + prompt = ( + f"Identify the Instagram screen layout type based on these DOM structural signals.\n" + f"Valid types: {[t.name for t in ScreenType]}\n" + f"Context:\n{layout_context}\n" + f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches." + ) + + try: + response = query_llm( + url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False + ) + if response and isinstance(response, str): + result = response.strip().upper() + elif response and isinstance(response, dict) and "response" in response: + result = response["response"].strip().upper() + else: + return ScreenType.UNKNOWN + + for t in ScreenType: + if t.name in result: + if signature and self.screen_memory: + self.screen_memory.store_screen(signature, t.name) + return t + except Exception as e: + import logging + + logging.getLogger(__name__).debug(f"LLM Classification failed: {e}") + + return ScreenType.UNKNOWN + + def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type): + """Discover what actions are possible on this screen.""" + actions = [] + + # Navigation tabs (always available when visible) + tab_map = { + "feed_tab": "tap home tab", + "search_tab": "tap explore tab", + "clips_tab": "tap reels tab", + "profile_tab": "tap profile tab", + "direct_tab": "tap messages tab", + } + for tab_id, action in tab_map.items(): + if tab_id in resource_ids: + actions.append(action) + + # Screen-specific actions + desc_lower = " ".join(content_descs).lower() + text_lower = " ".join(texts).lower() + + if "like" in desc_lower: + actions.append("tap like button") + if "comment" in desc_lower: + actions.append("tap comment button") + if "share" in desc_lower: + actions.append("tap share button") + if "save" in desc_lower or "bookmark" in desc_lower: + actions.append("tap save button") + if "back" in desc_lower: + actions.append("tap back button") + if any("follow" in e.get("text", "").lower() for e in clickable_elements): + actions.append("tap follow button") + + if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE: + if "message" in desc_lower or "nachricht" in desc_lower: + actions.append("tap message button") + if ( + "following" in desc_lower + or "abonniert" in desc_lower + or "following" in text_lower + or "profile_header_following" in " ".join(resource_ids).lower() + ): + actions.append("tap following list") + + # Grid items + if screen_type == ScreenType.EXPLORE_GRID: + actions.append("tap first grid item") + + # Scroll + actions.append("scroll down") + actions.append("press back") + + return list(set(actions)) # Deduplicate + + def _extract_context(self, content_descs, texts, resource_ids, screen_type): + """Extract meaningful context from the screen.""" + context = {} + + desc_text = " ".join(content_descs) + + # Username on profile + username_match = re.search(r"(\w+)'s (?:profile|story|unseen story)", desc_text) + if username_match: + context["username"] = username_match.group(1) + + # Post/follower counts + for d in content_descs: + m = re.match(r"([\d,.]+K?M?)(\s*)(posts?|followers?|following)", d, re.IGNORECASE) + if m: + context[m.group(3).lower()] = m.group(1) + + # Like state + for d in content_descs: + if d.lower() == "liked": + context["is_liked"] = True + elif d.lower() == "like": + context["is_liked"] = False + + return context + + def _compute_signature(self, resource_ids, content_descs, texts): + """Compute a stable hash for this screen state (for Qdrant lookup).""" + # Use sorted IDs + key content for stability + sig_parts = sorted(resource_ids)[:20] + sig_parts.extend(sorted(set(d.lower()[:30] for d in content_descs if len(d) > 2))[:10]) + sig = "|".join(sig_parts) + return hashlib.sha256(sig.encode()).hexdigest()[:24] + + def _empty_screen(self): + return { + "screen_type": ScreenType.FOREIGN_APP, + "available_actions": ["press back", "force start instagram"], + "selected_tab": None, + "context": {}, + "signature": "empty", + } diff --git a/GramAddict/core/perception/semantic_evaluator.py b/GramAddict/core/perception/semantic_evaluator.py new file mode 100644 index 0000000..e502e24 --- /dev/null +++ b/GramAddict/core/perception/semantic_evaluator.py @@ -0,0 +1,144 @@ +import json +import logging +import re +from typing import List, Optional + +from GramAddict.core.llm_provider import query_telepathic_llm +from GramAddict.core.perception.spatial_parser import SpatialNode + +logger = logging.getLogger(__name__) + + +class SemanticEvaluator: + """ + Handles LLM/VLM interaction for high-level semantic analysis of the UI. + Delegates vision processing and prompt engineering out of the core routing engine. + """ + + def __init__(self): + from GramAddict.core.config import Config + + try: + self.args = Config().args + except Exception: + self.args = None + + def _query_vlm(self, prompt: str, screenshot_b64: str) -> Optional[str]: + if not self.args: + logger.warning("👁️ [Vision Core] No config available. Cannot query VLM.") + return None + + model = getattr(self.args, "ai_telepathic_model", "llama3.2-vision") + url = getattr(self.args, "ai_telepathic_url", "http://localhost:11434/api/generate") + + try: + res = query_telepathic_llm( + model=model, + url=url, + system_prompt="You are an expert Instagram assistant.", + user_prompt=prompt, + images_b64=[screenshot_b64], + ) + return res + except Exception as e: + logger.error(f"👁️ [Vision Core] LLM query failed: {e}") + return None + + def evaluate_grid_visuals( + self, device, persona_interests: list[str], grid_nodes: List[SpatialNode] + ) -> Optional[SpatialNode]: + """ + Takes the spatial grid nodes and asks the VLM which one best matches the persona. + """ + logger.info(f"👁️ [Vision Core] Analyzing grid aesthetics against niche interests: {persona_interests}...") + + if not grid_nodes: + return None + + # Take a screenshot + try: + screenshot_b64 = device.get_screenshot_b64() + except Exception as e: + logger.error(f"👁️ [Vision Core] Failed to capture screenshot: {e}") + return None + + simplified_nodes = [] + for i, node in enumerate(grid_nodes[:9]): # Limit to 9 to save tokens + simplified_nodes.append({"index": i, "bounds": node.bounds}) + + prompt = f""" + You are a highly perceptive Instagram user with the following interests: {', '.join(persona_interests)}. + Look at the provided screenshot of the Instagram Explore/Profile grid. + Below are the bounding boxes for the top grid posts currently visible. + + {simplified_nodes} + + Your task: + 1. Identify which of these posts visually aligns BEST with your interests. + 2. Reply ONLY in JSON format: {{"index": }} + 3. If absolutely none of them are relevant, reply with {{"index": -1}}. + """ + + try: + response = self._query_vlm(prompt, screenshot_b64) + if not response: + return None + + try: + data = json.loads(response) + idx = data.get("index", -1) + if idx == -1: + logger.info("👁️ [Vision Core] VLM rejected all grid items. Will scroll down.") + return None + + if 0 <= idx < len(grid_nodes): + logger.info(f"👁️ [Vision Core] VLM selected grid item index [{idx}] as the best match.") + return grid_nodes[idx] + except json.JSONDecodeError: + # Fallback to fuzzy + clean_res = response.strip().upper() + match = re.search(r"\d+", clean_res) + if match: + idx = int(match.group()) + if 0 <= idx < len(grid_nodes): + logger.info(f"👁️ [Vision Core] VLM selected grid item index [{idx}] as the best match.") + return grid_nodes[idx] + except Exception as e: + logger.warning(f"👁️ [Vision Core] Exception during grid evaluation: {e}") + + return None + + def evaluate_post_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: + """Evaluates whether the currently viewed post aligns with persona interests.""" + logger.info(f"👁️ [Vision Core] Evaluating post vibe against: {persona_interests}") + try: + screenshot_b64 = device.get_screenshot_b64() + prompt = f""" + You are a user with the following interests: {', '.join(persona_interests)}. + You are looking at an Instagram post. + Evaluate if this post is highly relevant to your interests and if you should like/comment on it. + + Reply ONLY in valid JSON format: + {{ + "should_like": true/false, + "should_comment": true/false, + "reasoning": "brief explanation" + }} + """ + response = self._query_vlm(prompt, screenshot_b64) + if response: + if "```json" in response: + json_str = response.split("```json")[1].split("```")[0].strip() + else: + json_str = response.strip() + return json.loads(json_str) + except Exception as e: + logger.warning(f"Failed to evaluate post vibe: {e}") + return None + + def evaluate_profile_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: + """Evaluates if a profile is worth following.""" + pass + + def classify_screen_content(self, xml_hierarchy: str, target_class: str) -> Optional[str]: + pass diff --git a/GramAddict/core/perception/spatial_parser.py b/GramAddict/core/perception/spatial_parser.py new file mode 100644 index 0000000..1c1dba9 --- /dev/null +++ b/GramAddict/core/perception/spatial_parser.py @@ -0,0 +1,194 @@ +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + + +@dataclass +class SpatialNode: + """A single node in the Spatial Graph, representing a UI element and its geometry.""" + + bounds: Tuple[int, int, int, int] # (x1, y1, x2, y2) + node_id: str = "" + class_name: str = "" + text: str = "" + content_desc: str = "" + resource_id: str = "" + clickable: bool = False + scrollable: bool = False + + # Spatial Properties + children: List["SpatialNode"] = field(default_factory=list) + parent: Optional["SpatialNode"] = None + + @property + def x1(self) -> int: + return self.bounds[0] + + @property + def y1(self) -> int: + return self.bounds[1] + + @property + def x2(self) -> int: + return self.bounds[2] + + @property + def y2(self) -> int: + return self.bounds[3] + + @property + def width(self) -> int: + return self.x2 - self.x1 + + @property + def height(self) -> int: + return self.y2 - self.y1 + + @property + def center_x(self) -> int: + return self.x1 + (self.width // 2) + + @property + def center_y(self) -> int: + return self.y1 + (self.height // 2) + + @property + def area(self) -> int: + return self.width * self.height + + def contains(self, other: "SpatialNode") -> bool: + """Returns True if this node completely encompasses the other node geometrically.""" + return self.x1 <= other.x1 and self.y1 <= other.y1 and self.x2 >= other.x2 and self.y2 >= other.y2 + + def intersects(self, other: "SpatialNode") -> bool: + """Returns True if this node's bounding box overlaps with the other's bounding box.""" + if self.x1 >= other.x2 or other.x1 >= self.x2: + return False + if self.y1 >= other.y2 or other.y1 >= self.y2: + return False + return True + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.node_id, + "class": self.class_name, + "text": self.text, + "content_desc": self.content_desc, + "resource_id": self.resource_id, + "bounds": self.bounds, + "clickable": self.clickable, + "scrollable": self.scrollable, + "center": (self.center_x, self.center_y), + } + + +class SpatialParser: + """ + Parses Android UI XML into a structured 2D Spatial Tree. + Calculates parent-child relationships structurally, not just based on XML nesting. + """ + + def __init__(self): + self._node_counter = 0 + + def parse(self, xml_string: str) -> Optional[SpatialNode]: + """Parses the raw XML dump into a Spatial Graph.""" + try: + clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_string).strip() + if not clean_xml: + return None + root_elem = ET.fromstring(clean_xml) + + # 1. First Pass: Create flat list of spatial nodes + all_nodes = [] + self._flatten_xml(root_elem, all_nodes) + + if not all_nodes: + return None + + # 2. Second Pass: Reconstruct tree based on strict spatial containment + # Sort nodes by area descending (largest first) + all_nodes.sort(key=lambda n: n.area, reverse=True) + + root_node = all_nodes[0] + + for i in range(1, len(all_nodes)): + child = all_nodes[i] + # Find the smallest node that contains this child + # Since we sorted by area descending, we search backwards to find the tightest fit + parent_found = False + for j in range(i - 1, -1, -1): + potential_parent = all_nodes[j] + if potential_parent.contains(child): + potential_parent.children.append(child) + child.parent = potential_parent + parent_found = True + break + + # Fallback to root if no parent found (floating node) + if not parent_found and child != root_node: + root_node.children.append(child) + child.parent = root_node + + return root_node + + except ET.ParseError: + return None + + def _flatten_xml(self, element: ET.Element, nodes_list: List[SpatialNode]): + """Recursively traverses the XML and creates a flat list of SpatialNodes.""" + attrib = element.attrib + + bounds_str = attrib.get("bounds", "") + match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str) + + if match: + left, top, right, bottom = map(int, match.groups()) + + # Filter zero-area nodes early + if right > left and bottom > top: + self._node_counter += 1 + node = SpatialNode( + node_id=f"n_{self._node_counter}", + class_name=attrib.get("class", ""), + text=attrib.get("text", "").strip(), + content_desc=attrib.get("content-desc", "").strip(), + resource_id=attrib.get("resource-id", "").strip(), + bounds=(left, top, right, bottom), + clickable=attrib.get("clickable", "false") == "true", + scrollable=attrib.get("scrollable", "false") == "true", + ) + nodes_list.append(node) + + for child in element: + self._flatten_xml(child, nodes_list) + + def get_all_nodes(self, root: SpatialNode) -> List[SpatialNode]: + """Flattens the Spatial Tree into a list for easy filtering.""" + result = [root] + for child in root.children: + result.extend(self.get_all_nodes(child)) + return result + + def get_clickable_nodes(self, root: SpatialNode) -> List[SpatialNode]: + """Returns all nodes that are clickable or have strong semantic meaning.""" + all_nodes = self.get_all_nodes(root) + clickables = [] + + for n in all_nodes: + has_semantic = bool(n.text or n.content_desc) + semantic_res = n.resource_id and any( + x in n.resource_id.lower() for x in ["button", "tab", "icon", "action", "menu"] + ) + + if n.clickable or n.scrollable or semantic_res or (has_semantic and n.area < 500000 and n.area > 0): + # Filter out pure massive containers (like whole screen) if they aren't explicitly clickable + if not n.clickable and not n.scrollable and n.area > 2000000: + continue + # Also exclude if it's just a ViewGroup with a description but no action + if not n.clickable and n.class_name == "android.view.ViewGroup": + continue + clickables.append(n) + + return clickables diff --git a/GramAddict/core/qdrant_memory.py b/GramAddict/core/qdrant_memory.py index f992135..6aaef49 100644 --- a/GramAddict/core/qdrant_memory.py +++ b/GramAddict/core/qdrant_memory.py @@ -1,20 +1,21 @@ -import os -import json -import logging -import re import hashlib +import logging +import os +import re import time -from typing import Optional -from qdrant_client import QdrantClient -from qdrant_client.models import VectorParams, Distance, PointStruct, Filter, FieldCondition, MatchValue -from dotenv import load_dotenv -import requests import uuid +from typing import Optional + +import requests +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, FieldCondition, Filter, MatchValue, PointStruct, VectorParams logger = logging.getLogger(__name__) + class QdrantBase: _connection_failed_logged = False + def __init__(self, collection_name, vector_size=768): if QdrantBase._connection_failed_logged: self.client = None @@ -25,11 +26,11 @@ class QdrantBase: self._vector_size = vector_size self._consecutive_errors = 0 self._circuit_open = False - + try: qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344") self.client = QdrantClient(url=qdrant_url, timeout=10.0) - + if self.client: if self.client.collection_exists(collection_name): # Verify dimension match @@ -47,7 +48,9 @@ class QdrantBase: collection_name=collection_name, vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE), ) - logger.info(f"Recreated Qdrant collection '{collection_name}' with dimension {vector_size}.") + logger.info( + f"Recreated Qdrant collection '{collection_name}' with dimension {vector_size}." + ) except Exception as dim_err: logger.debug(f"Qdrant dimension check failed (non-fatal): {dim_err}") else: @@ -58,7 +61,10 @@ class QdrantBase: logger.info(f"Created Qdrant collection '{collection_name}' with dimension {vector_size}.") except Exception as e: if not QdrantBase._connection_failed_logged: - logger.error(f"Failed to initialize Qdrant memory (likely not running): {e}") + if "ConnectError" in str(type(e)): + logger.error(f"⚠️ [Qdrant] Connection refused at {qdrant_url}. Is Qdrant running?") + else: + logger.error(f"Failed to initialize Qdrant memory (likely not running): {e}") QdrantBase._connection_failed_logged = True # No debug logs here to prevent noise in --debug mode self.client = None @@ -86,7 +92,9 @@ class QdrantBase: logger.error(f"❌ [Qdrant] {context}: {e}") if self._consecutive_errors >= 3: if not self._circuit_open: - logger.critical(f"🚨 [Qdrant] Circuit breaker triggered! Disabling memory engine for {self.collection_name} due to persistent failures.") + logger.critical( + f"🚨 [Qdrant] Circuit breaker triggered! Disabling memory engine for {self.collection_name} due to persistent failures." + ) self._circuit_open = True def _handle_success(self): @@ -96,43 +104,39 @@ class QdrantBase: self._circuit_open = False def _get_embedding(self, text: str) -> Optional[list]: - from GramAddict.core.config import Config + if not hasattr(self, "_cached_args"): cfg = Config() self._cached_args = cfg.args if hasattr(cfg, "args") else None args = self._cached_args - + # Pull specific embedding config or fallback to defaults model = getattr(args, "ai_embedding_model", "nomic-embed-text") url = getattr(args, "ai_embedding_url", "http://localhost:11434/api/embeddings") - + try: # Generate embeddings is_cloud = "openrouter.ai" in url.lower() or "openai.com" in url.lower() - + headers = {} if is_cloud: key = os.environ.get("OPENROUTER_API_KEY") if key: headers["Authorization"] = f"Bearer {key}" # OpenAI/OpenRouter use 'input' instead of 'prompt' - payload = { - "model": model, - "input": str(text)[:8000] - } + payload = {"model": model, "input": str(text)[:8000]} else: # Local Ollama - payload = { - "model": model, - "prompt": str(text)[:8000] - } - + payload = {"model": model, "prompt": str(text)[:8000]} + # Log to prevent user from thinking the bot is hung during model swap in VRAM if not getattr(self, "_has_logged_embedding", False): - logger.debug(f"🧠 [Ollama] Generating embeddings... (First local hit may take 5-10s to load model into VRAM)") + logger.debug( + "🧠 [Ollama] Generating embeddings... (First local hit may take 5-10s to load model into VRAM)" + ) self._has_logged_embedding = True - + resp = requests.post( url, json=payload, @@ -142,7 +146,7 @@ class QdrantBase: if resp.status_code != 200: logger.debug(f"Embedding API Error {resp.status_code}: {resp.text}") resp.raise_for_status() - + data = resp.json() # Handle both Ollama and OpenAI-style embedding responses if "embedding" in data: @@ -160,7 +164,8 @@ class QdrantBase: """ import hashlib import uuid - return str(uuid.UUID(hex=hashlib.sha256(seed_string.encode('utf-8')).hexdigest()[:32])) + + return str(uuid.UUID(hex=hashlib.sha256(seed_string.encode("utf-8")).hexdigest()[:32])) def upsert_point(self, seed_string: str, payload: dict, vector: list = None, log_success: str = None) -> bool: """ @@ -168,26 +173,46 @@ class QdrantBase: """ if not self.is_connected: return False - + point_id = self.generate_uuid(seed_string) - + # If no vector provided, use a zero-vector if expected, though Qdrant requires vectors unless using empty vectors config. # Most of our DBs provide vectors, or we just pass a list of 0.0s. safe_vector = vector if vector is not None else [0.0] * self._vector_size - + try: self.client.upsert( collection_name=self.collection_name, - points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)] + points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)], ) - + # ABSOLUTE LOGGING: User requirement for full observability msg = log_success or f"📥 [Qdrant] Learned new data in {self.collection_name}" logger.info(f"{msg} (UUID: {point_id[:8]}...)", extra={"color": "\x1b[35m"}) self._handle_success() return True except Exception as e: - self._handle_error(e, f"Failed to upsert point") + self._handle_error(e, "Failed to upsert point") + return False + + def delete_point(self, seed_string: str) -> bool: + """ + Deletes a point from Qdrant autonomously based on its seed string. + Used by the Unlearn mechanism to purge false-positive memories. + """ + if not self.is_connected: + return False + + point_id = self.generate_uuid(seed_string) + try: + self.client.delete(collection_name=self.collection_name, points_selector=[point_id]) + logger.info( + f"🗑️ [Qdrant] Purged poisoned memory vector from {self.collection_name} (UUID: {point_id[:8]}...)", + extra={"color": "\x1b[31m"}, + ) + return True + except Exception as e: + self._handle_error(e, f"Failed to delete point {point_id}") return False def search_points(self, vector: list, limit: int = 5, score_threshold: float = 0.5) -> list: @@ -196,20 +221,18 @@ class QdrantBase: """ if not self.is_connected: return [] - + try: results = self.client.search( - collection_name=self.collection_name, - query_vector=vector, - limit=limit, - score_threshold=score_threshold + collection_name=self.collection_name, query_vector=vector, limit=limit, score_threshold=score_threshold ) logger.debug(f"🔍 [Qdrant] Search in {self.collection_name} returned {len(results)} matches.") self._handle_success() return results except Exception as e: - self._handle_error(e, f"Search failed") + self._handle_error(e, "Search failed") return [] + def get_collection_size(self) -> int: """Returns the number of points in the collection.""" if not self.is_connected: @@ -226,16 +249,19 @@ class HeuristicMemoryDB(QdrantBase): Stores successfully generated heuristics from the VLMCompilerEngine. This allows the ZeroLatencyEngine to pull pre-compiled Regex/XPath rules securely. """ + def __init__(self): super().__init__(collection_name="gramaddict_heuristics_v7") def cache_heuristic(self, intent_description: str, rule: dict): - if not self.is_connected or not rule: return + if not self.is_connected or not rule: + return try: # We hash the intent description to create a deterministic PointID doc_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, intent_description)) vector = self._get_embedding(intent_description) - if not vector: return + if not vector: + return payload = { "intent": intent_description, @@ -243,32 +269,34 @@ class HeuristicMemoryDB(QdrantBase): "target_attribute": rule.get("target_attribute", "text"), "pattern": rule.get("pattern", ""), "timestamp": time.time(), - "confidence": 0.9 # Initial confidence + "confidence": 0.9, # Initial confidence } - + self.upsert_point(intent_description, payload, vector=vector) except Exception as e: logger.error(f"Failed to cache generated heuristic: {e}") def fetch_heuristic(self, intent_description: str) -> Optional[dict]: - if not self.is_connected: return None + if not self.is_connected: + return None try: vector = self._get_embedding(intent_description) - if not vector: return None - + if not vector: + return None + results = self.client.query_points( collection_name=self.collection_name, query=vector, limit=1, - score_threshold=0.98 # Very strict match on intent description + score_threshold=0.98, # Very strict match on intent description ) if results and getattr(results, "points", None): payload = results.points[0].payload return { "rule_type": payload.get("rule_type"), "target_attribute": payload.get("target_attribute"), - "pattern": payload.get("pattern") + "pattern": payload.get("pattern"), } logger.debug(f"fetch_heuristic: No points found matching {intent_description} above threshold.") return None @@ -276,17 +304,18 @@ class HeuristicMemoryDB(QdrantBase): logger.debug(f"Failed to fetch heuristic for {intent_description}: {e}") return None + class UIMemoryDB(QdrantBase): """ Hardened Qdrant-based UI memory for storing and retrieving learned element locations. - + Key design decisions: - Deterministic IDs: Hash of (intent + structural_signature) → same situation overwrites old entry - Confidence scoring: Entries gain/lose confidence based on success/failure feedback - No negative caching: NOT_FOUND results are NEVER stored (prevents memory poisoning) - Post-validation: Retrieved entries are checked for staleness """ - + # Minimum confidence score to return a cached result MIN_CONFIDENCE = 0.3 # Default confidence for new entries @@ -306,18 +335,29 @@ class UIMemoryDB(QdrantBase): # We keep class and resource-id as core structural identifiers. # We remove bounds as they change with resolution/scaling and bloat the string. attributes_to_remove = [ - 'text', 'content-desc', 'bounds', 'checkable', 'clickable', - 'focusable', 'focused', 'scrollable', 'long-clickable', - 'password', 'selected', 'index', 'package', 'instance' + "text", + "content-desc", + "bounds", + "checkable", + "clickable", + "focusable", + "focused", + "scrollable", + "long-clickable", + "password", + "selected", + "index", + "package", + "instance", ] - + sig = xml_context for attr in attributes_to_remove: - sig = re.sub(rf'{attr}="[^"]*"', '', sig) - + sig = re.sub(rf'{attr}="[^"]*"', "", sig) + # 2. Cleanup whitespace - sig = re.sub(r'\s+', ' ', sig).strip() - + sig = re.sub(r"\s+", " ", sig).strip() + # 3. Strict truncation for nomic-embed-text context window return sig[:4000] @@ -328,7 +368,9 @@ class UIMemoryDB(QdrantBase): """ return hashlib.sha256(intent.encode("utf-8")).hexdigest()[:32] - def retrieve_memory(self, intent: str, xml_context: str, similarity_threshold: float = None, exact_only: bool = False) -> Optional[dict]: + def retrieve_memory( + self, intent: str, xml_context: str, similarity_threshold: float = None, exact_only: bool = False + ) -> Optional[dict]: """ Queries Qdrant for a known resolution representing the given intent. Returns the cached intent result (e.g. is_ad boolean, or bounding box) if found. @@ -343,31 +385,34 @@ class UIMemoryDB(QdrantBase): def _evaluate_payload(payload: dict, score: float = 1.0, point_id: str = None) -> Optional[dict]: solution = payload.get("solution") confidence = payload.get("confidence", self.DEFAULT_CONFIDENCE) - + # ── TTL-based confidence decay ── stored_at = payload.get("stored_at", 0) if stored_at > 0: import time as _time + age_hours = (_time.time() - stored_at) / 3600 time_decay = min(age_hours * 0.05, 0.4) # Max 40% decay over 8 hours effective_confidence = confidence - time_decay else: effective_confidence = confidence - + # Filter 1: Never return NOT_FOUND/error entries from cache if isinstance(solution, dict) and solution.get("type") == "error": logger.debug(f"Purging poisoned NOT_FOUND entry for '{intent}' from Qdrant Memory.") if point_id: self._delete_point(point_id) return None - + # Filter 2: Skip low-confidence entries if effective_confidence < self.MIN_CONFIDENCE: - logger.debug(f"Skipping expired/low-confidence ({effective_confidence:.2f}, raw: {confidence:.2f}) entry for '{intent}'.") + logger.debug( + f"Skipping expired/low-confidence ({effective_confidence:.2f}, raw: {confidence:.2f}) entry for '{intent}'." + ) if effective_confidence < 0.1 and point_id: self._delete_point(point_id) return None - + return {"solution": solution, "effective_confidence": effective_confidence} # 0. Zero-cost deterministic ID exact match (bypasses LLM embedding API) @@ -382,7 +427,9 @@ class UIMemoryDB(QdrantBase): if exact_points: eval_result = _evaluate_payload(exact_points[0].payload, score=1.0, point_id=point_id) if eval_result: - logger.debug(f"Resolved intent '{intent}' from Qdrant Memory via EXACT ID MATCH! (Confidence: {eval_result['effective_confidence']:.2f})") + logger.debug( + f"Resolved intent '{intent}' from Qdrant Memory via EXACT ID MATCH! (Confidence: {eval_result['effective_confidence']:.2f})" + ) return eval_result["solution"] # If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent! return None @@ -396,7 +443,7 @@ class UIMemoryDB(QdrantBase): # Embed ONLY the intent. We learn elements globally to drastically reduce LLM calls. text_to_embed = f"Intent: {intent}" vector = self._get_embedding(text_to_embed) - + if not vector: return None @@ -410,7 +457,9 @@ class UIMemoryDB(QdrantBase): if results and results[0].score >= similarity_threshold: eval_result = _evaluate_payload(results[0].payload, score=results[0].score, point_id=results[0].id) if eval_result: - logger.debug(f"Resolved intent '{intent}' from Qdrant Memory via vector search! (Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})") + logger.debug( + f"Resolved intent '{intent}' from Qdrant Memory via vector search! (Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})" + ) return eval_result["solution"] return None else: @@ -436,14 +485,14 @@ class UIMemoryDB(QdrantBase): text_to_embed = f"Intent: {intent}" vector = self._get_embedding(text_to_embed) - + if not vector: return try: # Deterministic ID: same intent = overwrite, not duplicate point_id = self._deterministic_id(intent) - + self.client.upsert( collection_name=self.collection_name, points=[ @@ -455,10 +504,10 @@ class UIMemoryDB(QdrantBase): "solution": solution, "confidence": self.DEFAULT_CONFIDENCE, "stored_at": time.time(), - } + }, ) ], - wait=True + wait=True, ) logger.info(f"Learned pattern for '{intent}' and saved to Qdrant Memory (ID: {point_id[:8]}...).") except Exception as e: @@ -468,18 +517,24 @@ class UIMemoryDB(QdrantBase): """Called when an element from memory was successfully used.""" intent = kwargs.get("intent") or (args[0] if args else None) amount = kwargs.get("amount", 0.15) - if not intent: return - try: amount = float(amount) - except (ValueError, TypeError): amount = 0.15 + if not intent: + return + try: + amount = float(amount) + except (ValueError, TypeError): + amount = 0.15 self._adjust_confidence(intent, amount) def decay_confidence(self, *args, **kwargs): """Called when an element from memory failed validation (not found on screen).""" intent = kwargs.get("intent") or (args[0] if args else None) amount = kwargs.get("amount", 0.50) # Harder punishment: 2 failures = near-dead - if not intent: return - try: amount = float(amount) - except (ValueError, TypeError): amount = 0.50 + if not intent: + return + try: + amount = float(amount) + except (ValueError, TypeError): + amount = 0.50 self._adjust_confidence(intent, -amount) def _adjust_confidence(self, intent: str, delta: float): @@ -488,7 +543,7 @@ class UIMemoryDB(QdrantBase): return point_id = self._deterministic_id(intent) - + try: # Fetch current point points = self.client.retrieve( @@ -497,19 +552,19 @@ class UIMemoryDB(QdrantBase): with_payload=True, with_vectors=False, ) - + if not points: return - + current = points[0].payload new_confidence = max(0.0, min(1.0, current.get("confidence", self.DEFAULT_CONFIDENCE) + delta)) - + # If confidence dropped below threshold, delete the entry entirely if new_confidence < 0.1: logger.info(f"Confidence for '{intent}' dropped to {new_confidence:.2f}. Removing stale entry.") self._delete_point(point_id) return - + # Update payload with new confidence self.client.set_payload( collection_name=self.collection_name, @@ -524,6 +579,7 @@ class UIMemoryDB(QdrantBase): """Deletes a single point from Qdrant by ID.""" try: from qdrant_client.models import PointIdsList + self.client.delete( collection_name=self.collection_name, points_selector=PointIdsList(points=[point_id]), @@ -543,13 +599,14 @@ class UIMemoryDB(QdrantBase): ) purged = 0 import time as _time + now = _time.time() for pt in points: payload = pt.payload or {} confidence = payload.get("confidence", self.DEFAULT_CONFIDENCE) stored_at = payload.get("stored_at", 0) age_hours = (now - stored_at) / 3600 if stored_at > 0 else 0 - + # Purge if: confidence dropped below 0.5, or older than 24 hours if confidence < 0.5 or age_hours > 24: self._delete_point(pt.id) @@ -576,12 +633,8 @@ class CommentMemoryDB(QdrantBase): self.upsert_point( seed_string=str(time.time()), vector=vector, - payload={ - "text": text, - "vibe": vibe, - "author": author - }, - log_success=f"🧠 [CommentMemory] Learned and stored comment: '{text[:30]}...' (Vibe: {vibe})" + payload={"text": text, "vibe": vibe, "author": author}, + log_success=f"🧠 [CommentMemory] Learned and stored comment: '{text[:30]}...' (Vibe: {vibe})", ) def get_relevant_comments(self, context_text: str, limit: int = 3) -> list[str]: @@ -604,6 +657,7 @@ class CommentMemoryDB(QdrantBase): logger.debug(f"Comment retrieval error: {e}") return [] + class ContentMemoryDB(QdrantBase): def __init__(self): super().__init__(collection_name="gramaddict_content_memory") @@ -626,7 +680,7 @@ class ContentMemoryDB(QdrantBase): "reason": reason, "stored_at": time.time(), }, - log_success=f"🧠 [ContentMemory] Stored evaluation in memory: {classification} ({reason})" + log_success=f"🧠 [ContentMemory] Stored evaluation in memory: {classification} ({reason})", ) def get_cached_evaluation(self, description: str, similarity_threshold: float = 0.95) -> Optional[dict]: @@ -647,7 +701,9 @@ class ContentMemoryDB(QdrantBase): if results and results[0].score >= similarity_threshold: payload = results[0].payload - logger.info(f"Qdrant Cache Hit! Similar post found (score: {results[0].score:.3f}). Classification: {payload.get('classification')}") + logger.info( + f"Qdrant Cache Hit! Similar post found (score: {results[0].score:.3f}). Classification: {payload.get('classification')}" + ) return payload return None except Exception as e: @@ -669,25 +725,29 @@ class ContentMemoryDB(QdrantBase): query=vector, limit=limit, ).points - + examples = [] for hit in results: payload = hit.payload - examples.append({ - "description": payload.get("description", "")[:200], - "classification": payload.get("classification"), - "reason": payload.get("reason"), - }) + examples.append( + { + "description": payload.get("description", "")[:200], + "classification": payload.get("classification"), + "reason": payload.get("reason"), + } + ) return examples except Exception as e: logger.debug(f"Content RAG retrieval error: {e}") return [] + class ScreenMemoryDB(QdrantBase): """ Learns and caches structural screen classifications mapping (XML Signature -> ScreenType). Replaces hardcoded string checks in GOAP. """ + def __init__(self): super().__init__(collection_name="gramaddict_screen_types_v1") @@ -707,7 +767,7 @@ class ScreenMemoryDB(QdrantBase): "screen_type": screen_type, "stored_at": time.time(), }, - log_success=f"🧠 [ScreenMemory] Learned new layout mapping: {screen_type}" + log_success=f"🧠 [ScreenMemory] Learned new layout mapping: {screen_type}", ) def get_screen_type(self, xml_signature: str, similarity_threshold: float = 0.90) -> Optional[str]: @@ -727,36 +787,49 @@ class ScreenMemoryDB(QdrantBase): if results and results[0].score >= similarity_threshold: payload = results[0].payload - logger.info(f"🧠 [ScreenMemory] Cache Hit! Screen recognized as: {payload.get('screen_type')} (Score: {results[0].score:.2f})") + logger.info( + f"🧠 [ScreenMemory] Cache Hit! Screen recognized as: {payload.get('screen_type')} (Score: {results[0].score:.2f})" + ) return payload.get("screen_type") return None except Exception as e: logger.debug(f"Screen memory error: {e}") return None + def purge_screen(self, xml_signature: str): + """Unlearns a screen classification to recover from hallucinations.""" + if not self.is_connected or not xml_signature: + return + self.delete_point(xml_signature) + + class NavigationMemoryDB(QdrantBase): """ Topological Navigation Persistence. Stores learned paths and transitions discovered during autonomous exploration. This eliminates "Context Stagnation" by sharing learned navigation across the fleet. """ + def __init__(self): # We use a smaller vector size for navigation tags or just zero-vectors for KV storage super().__init__(collection_name="gramaddict_nav_graph_v8", vector_size=128) def store_transition(self, from_state: str, action: str, to_state: str): key = f"{from_state}_{action}" - payload = { - "from": from_state, - "action": action, - "to": to_state, - "timestamp": time.time() - } - self.upsert_point(key, payload, log_success=f"🧠 [NavBrain] Learned transition: {from_state} --({action})--> {to_state}") + payload = {"from": from_state, "action": action, "to": to_state, "timestamp": time.time()} + self.upsert_point( + key, payload, log_success=f"🧠 [NavBrain] Learned transition: {from_state} --({action})--> {to_state}" + ) + + def unlearn_transition(self, from_state: str, action: str): + """Unlearns a topological transition if it is proven to be a dead-end.""" + key = f"{from_state}_{action}" + self.delete_point(key) def get_all_transitions(self) -> dict: """Retrieves the full topological map from Qdrant.""" - if not self.is_connected: return {} + if not self.is_connected: + return {} try: resp = self.client.scroll(collection_name=self.collection_name, limit=100) points = resp[0] @@ -764,18 +837,19 @@ class NavigationMemoryDB(QdrantBase): for p in points: data = p.payload f, a, t = data["from"], data["action"], data["to"] - if f not in nodes: nodes[f] = {"transitions": {}} + if f not in nodes: + nodes[f] = {"transitions": {}} nodes[f]["transitions"][a] = t return nodes except Exception as e: logger.error(f"Failed to fetch nav graph from Qdrant: {e}") return {} -class PersonaMemoryDB(QdrantBase): +class PersonaMemoryDB(QdrantBase): def __init__(self): super().__init__(collection_name="gramaddict_persona_memory") - + def store_persona_insight(self, category: str, insight: str): """Saves a learned insight about the logged-in user's brand persona.""" if not self.is_connected or not insight: @@ -793,7 +867,7 @@ class PersonaMemoryDB(QdrantBase): "insight": insight, "stored_at": time.time(), }, - log_success=f"🧠 [PersonaMemory] Learned Brand Persona ({category}): {insight[:50]}..." + log_success=f"🧠 [PersonaMemory] Learned Brand Persona ({category}): {insight[:50]}...", ) def get_persona_context(self, category: str = None, limit: int = 5) -> str: @@ -804,15 +878,8 @@ class PersonaMemoryDB(QdrantBase): try: query_filter = None if category: - query_filter = Filter( - must=[ - FieldCondition( - key="category", - match=MatchValue(value=category) - ) - ] - ) - + query_filter = Filter(must=[FieldCondition(key="category", match=MatchValue(value=category))]) + # Use scroll to just fetch the most recent or all relevant insights since we don't have a semantic query to match against. # We just want the general guidelines for the prompt. points, _ = self.client.scroll( @@ -821,35 +888,36 @@ class PersonaMemoryDB(QdrantBase): limit=limit, with_payload=True, ) - + if not points: return "" - + contexts = [] for pt in points: payload = pt.payload cat = payload.get("category", "General") ins = payload.get("insight", "") contexts.append(f"[{cat}] {ins}") - + return "\n".join(contexts) except Exception as e: logger.debug(f"Persona retrieval error: {e}") return "" + class BannedPathsDB: """ Negative feedback memory: stores which specific elements FAILED for which goals. Simple dict-based storage (no vectors needed). - + Key insight: This is NOT a vector similarity problem. The relationship is exact: "goal X + element Y = failure". So we use a simple hash map stored in Qdrant as payload-only points. """ - + # Max age for banned entries (7 days) — Instagram updates can change layouts MAX_AGE_SECONDS = 7 * 24 * 3600 - + def __init__(self): if QdrantBase._connection_failed_logged: self._client = None @@ -858,31 +926,32 @@ class BannedPathsDB: self._banned = {} # {goal_hash: set(element_ids)} self._client = None self._collection = "gramaddict_banned_paths" - + try: qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344") self._client = QdrantClient(url=qdrant_url, timeout=5.0) - + # Create collection with a dummy 1-dim vector (Qdrant requires vectors) if self._client and not self._client.collection_exists(self._collection): self._client.create_collection( collection_name=self._collection, vectors_config=VectorParams(size=4, distance=Distance.COSINE), ) - + # Load all banned entries into memory at startup self._load_all() except Exception as e: logger.debug(f"BannedPaths init (non-fatal): {e}") - + def _goal_hash(self, goal: str) -> str: return hashlib.sha256(goal.lower().strip().encode("utf-8")).hexdigest()[:16] - + def _point_id(self, goal: str, element_id: str) -> str: combined = f"{goal.lower().strip()}||{element_id}" import uuid + return str(uuid.UUID(hex=hashlib.sha256(combined.encode("utf-8")).hexdigest()[:32])) - + def _load_all(self): """Load all banned paths from Qdrant into memory at startup.""" if not self._client: @@ -907,34 +976,35 @@ class BannedPathsDB: if gh not in self._banned: self._banned[gh] = set() self._banned[gh].add(eid) - + # Clean expired if expired: from qdrant_client.models import PointIdsList + self._client.delete( collection_name=self._collection, points_selector=PointIdsList(points=expired), ) logger.debug(f"BannedPaths: Expired {len(expired)} old entries.") - + total = sum(len(v) for v in self._banned.values()) if total > 0: logger.info(f"BannedPaths: Loaded {total} banned element-goal pairs.") except Exception as e: logger.debug(f"BannedPaths load error (non-fatal): {e}") - + def ban(self, goal: str, element_id: str, reason: str = ""): """Ban a specific element for a specific goal.""" if not element_id: return - + gh = self._goal_hash(goal) if gh not in self._banned: self._banned[gh] = set() self._banned[gh].add(element_id) - + logger.warning(f"BannedPaths: Banned '{element_id}' for goal '{goal[:60]}'. Reason: {reason}") - + # Persist to Qdrant if self._client: try: @@ -951,37 +1021,38 @@ class BannedPathsDB: "element_id": element_id, "reason": reason[:200], "banned_at": time.time(), - } + }, ) - ] + ], ) except Exception as e: logger.debug(f"BannedPaths persist error: {e}") - + def is_banned(self, goal: str, element_id: str) -> bool: """Check if a specific element is banned for a specific goal.""" if not element_id: return False gh = self._goal_hash(goal) return element_id in self._banned.get(gh, set()) - + def get_banned_ids(self, goal: str) -> set: """Get all banned element IDs for a specific goal.""" gh = self._goal_hash(goal) return self._banned.get(gh, set()).copy() + class DMMemoryDB(QdrantBase): def __init__(self): super().__init__(collection_name="gramaddict_dm_memory") - + def log_sent_dm(self, target_username: str, message: str, biography: str, recent_descriptions: list): if not self.is_connected: return - + vector = self._get_embedding(message) if not vector: return - + self.upsert_point( seed_string=f"{target_username}_{message}_{time.time()}", vector=vector, @@ -992,11 +1063,11 @@ class DMMemoryDB(QdrantBase): "recent_descriptions": recent_descriptions, "status": "pending", "score": 0.0, - "timestamp": time.time() + "timestamp": time.time(), }, - log_success="🧠 [Growth Brain] Logged sent DM to memory for future response tracking." + log_success="🧠 [Growth Brain] Logged sent DM to memory for future response tracking.", ) - + def get_pending_dms(self, limit=10): if not self.is_connected: return [] @@ -1004,53 +1075,37 @@ class DMMemoryDB(QdrantBase): points = self.client.query_points( collection_name=self.collection_name, query=self._get_embedding(""), - query_filter=Filter( - must=[ - FieldCondition( - key="status", - match=MatchValue(value="pending") - ) - ] - ), - limit=limit + query_filter=Filter(must=[FieldCondition(key="status", match=MatchValue(value="pending"))]), + limit=limit, ).points - + return [p.payload for p in points] except Exception as e: logger.debug(f"[DMMemory] get_pending_dms failed: {e}") return [] - + def update_dm_score(self, point_id: str, new_score: float, status: str = "evaluated"): if not self.is_connected: return try: self.client.set_payload( - collection_name=self.collection_name, - payload={"status": status, "score": new_score}, - points=[point_id] + collection_name=self.collection_name, payload={"status": status, "score": new_score}, points=[point_id] ) logger.info(f"🧠 [Growth Brain] Updated DM Resonance Score: {new_score:.1f}") except Exception as e: logger.debug(f"Failed to update DM score: {e}") - + def get_best_performing_dms(self, limit=3): if not self.is_connected: return [] try: results = self.client.scroll( collection_name=self.collection_name, - scroll_filter=Filter( - must=[ - FieldCondition( - key="status", - match=MatchValue(value="evaluated") - ) - ] - ), + scroll_filter=Filter(must=[FieldCondition(key="status", match=MatchValue(value="evaluated"))]), limit=100, - with_payload=True + with_payload=True, )[0] - + dms = [r.payload["message"] for r in results if r.payload.get("score", 0.0) >= 0.8] dms = list(set(dms)) return dms[:limit] @@ -1058,45 +1113,39 @@ class DMMemoryDB(QdrantBase): logger.debug(f"Failed to fetch best DMs: {e}") return [] + class ParasocialCRMDB(QdrantBase): collection_name = "ig_parasocial_crm" - + def __init__(self): super().__init__(self.collection_name) - + def get_relationship_stage(self, username: str) -> dict: """ Returns the current CRM stage and last interaction timestamp. Stages: 0=Awareness, 1=Curiosity, 2=Rapport, 3=Conversion - + Uses exact-match filter (not embedding) — usernames are deterministic keys, not semantic content. This is ~100x faster than embedding search. """ if not self.is_connected: return {"stage": 0, "last_interaction": 0.0, "interactions": []} - + try: points, _ = self.client.scroll( collection_name=self.collection_name, - scroll_filter=Filter( - must=[ - FieldCondition( - key="username", - match=MatchValue(value=username) - ) - ] - ), + scroll_filter=Filter(must=[FieldCondition(key="username", match=MatchValue(value=username))]), limit=1, with_payload=True, ) - + if points: return points[0].payload except Exception as e: logger.debug(f"[ParasocialCRM] query failed: {e}") - + return {"stage": 0, "last_interaction": 0.0, "interactions": []} - + def log_interaction(self, username: str, intent_type: str, new_stage: int = None): """ Logs an interaction (e.g. 'story_view', 'deep_like', 'comment_reply'). @@ -1104,111 +1153,104 @@ class ParasocialCRMDB(QdrantBase): """ if not self.is_connected: return - + current = self.get_relationship_stage(username) stage = new_stage if new_stage is not None else current["stage"] - + interactions = current.get("interactions", []) import time + now = time.time() - + if interactions: last_interaction = interactions[-1] if last_interaction.get("type") == intent_type and (now - last_interaction.get("timestamp", 0)) < 300: logger.debug(f"🧠 [ParasocialCRM] Skipping redundant '{intent_type}' write for @{username}.") return - - interactions.append({ - "type": intent_type, - "timestamp": now - }) - + + interactions.append({"type": intent_type, "timestamp": now}) + payload = { "username": username, "stage": stage, "last_interaction": now, - "interactions": interactions[-10:] # keep last 10 + "interactions": interactions[-10:], # keep last 10 } - + vector = self._get_embedding(f"User: {username}") if vector: self.upsert_point( seed_string=f"User_{username}", vector=vector, payload=payload, - log_success=f"🧠 [ParasocialCRM] Updated @{username} into Qdrant. Stage {stage} ({intent_type})" + log_success=f"🧠 [ParasocialCRM] Updated @{username} into Qdrant. Stage {stage} ({intent_type})", ) def log_generated_comment(self, username: str, comment_text: str): """Phase 10: RAG memory point for specific users.""" if not self.is_connected: return - + current = self.get_relationship_stage(username) interactions = current.get("interactions", []) - - interactions.append({ - "type": "comment_sent", - "text": comment_text, - "timestamp": time.time() - }) - + + interactions.append({"type": "comment_sent", "text": comment_text, "timestamp": time.time()}) + payload = { "username": username, "stage": current["stage"], "last_interaction": time.time(), - "interactions": interactions[-20:], # Keep last 20 interactions - "bio": current.get("bio", "") + "interactions": interactions[-20:], # Keep last 20 interactions + "bio": current.get("bio", ""), } - + vector = self._get_embedding(f"User: {username}") if vector: self.upsert_point( seed_string=f"User_{username}", vector=vector, payload=payload, - log_success=f"🧠 [ParasocialCRM] Logged generated comment for @{username} into Qdrant." + log_success=f"🧠 [ParasocialCRM] Logged generated comment for @{username} into Qdrant.", ) - + def log_profile_context(self, username: str, bio: str): """Phase 10: Store parsed user bio for hyper-personalization.""" if not self.is_connected or not username: return - + current = self.get_relationship_stage(username) # Avoid unnecessary DB writes if the bio hasn't fundamentally changed if current.get("bio") == bio: return - + payload = { "username": username, "stage": current["stage"], "last_interaction": current["last_interaction"], "interactions": current.get("interactions", []), - "bio": bio + "bio": bio, } - + vector = self._get_embedding(f"User: {username}") if vector: self.upsert_point( seed_string=f"User_{username}", vector=vector, payload=payload, - log_success=f"🧠 [ParasocialCRM] Learned profile context for @{username} (Bio: {bio[:30]}...)" + log_success=f"🧠 [ParasocialCRM] Learned profile context for @{username} (Bio: {bio[:30]}...)", ) def get_conversation_context(self, username: str) -> str: """Phase 10: Constructs semantic context from prior comments with this user.""" current = self.get_relationship_stage(username) interactions = current.get("interactions", []) - + context_parts = [] if current.get("bio"): context_parts.append(f"User Bio: {current['bio']}") - + comments = [i["text"] for i in interactions if i.get("type") == "comment_sent" and "text" in i] if comments: context_parts.append("Previous Comments: " + " | ".join(comments)) - - return "\n".join(context_parts) + return "\n".join(context_parts) diff --git a/GramAddict/core/screen_topology.py b/GramAddict/core/screen_topology.py index 9544a49..cf2d6d1 100644 --- a/GramAddict/core/screen_topology.py +++ b/GramAddict/core/screen_topology.py @@ -8,8 +8,8 @@ This is the bot's GPS: it knows HOW to get from screen A to screen B before the bot starts moving. The GOAP planner consults this map as its primary routing strategy. """ + from collections import deque -from enum import Enum from typing import Dict, List, Optional, Tuple from GramAddict.core.goap import ScreenType @@ -38,6 +38,7 @@ class ScreenTopology: "tap home tab": ScreenType.HOME_FEED, "tap profile tab": ScreenType.OWN_PROFILE, "tap reels tab": ScreenType.REELS_FEED, + "view a post": ScreenType.POST_DETAIL, }, ScreenType.REELS_FEED: { "tap home tab": ScreenType.HOME_FEED, @@ -78,12 +79,16 @@ class ScreenTopology: "open messages": ScreenType.DM_INBOX, "open following list": ScreenType.FOLLOW_LIST, "open followers list": ScreenType.FOLLOW_LIST, + "view a post": ScreenType.POST_DETAIL, + "open post": ScreenType.POST_DETAIL, + "open post author profile": ScreenType.OTHER_PROFILE, + "view the user profile": ScreenType.OTHER_PROFILE, + "view user profile": ScreenType.OTHER_PROFILE, + "open user profile": ScreenType.OTHER_PROFILE, } @classmethod - def find_route( - cls, from_screen: ScreenType, to_screen: ScreenType - ) -> Optional[List[Tuple[str, ScreenType]]]: + def find_route(cls, from_screen: ScreenType, to_screen: ScreenType) -> Optional[List[Tuple[str, ScreenType]]]: """ BFS shortest path from from_screen to to_screen. @@ -171,9 +176,7 @@ class ScreenTopology: return f"navigate to {screen_name}" @classmethod - def expected_screen_for_action( - cls, action: str, from_screen: ScreenType - ) -> Optional[ScreenType]: + def expected_screen_for_action(cls, action: str, from_screen: ScreenType) -> Optional[ScreenType]: """What screen should we land on after this action from this screen? Used by _execute_action to validate INTERMEDIATE navigation steps. diff --git a/GramAddict/core/session_state.py b/GramAddict/core/session_state.py index c88d54b..3caf81f 100644 --- a/GramAddict/core/session_state.py +++ b/GramAddict/core/session_state.py @@ -80,64 +80,40 @@ class SessionState: self, ): """set the limits for current session""" - self.args.current_likes_limit = get_value( - getattr(self.args, "total_likes_limit", 300), None, 300 - ) - self.args.current_follow_limit = get_value( - getattr(self.args, "total_follows_limit", 50), None, 50 - ) - self.args.current_unfollow_limit = get_value( - getattr(self.args, "total_unfollows_limit", 50), None, 50 - ) - self.args.current_comments_limit = get_value( - getattr(self.args, "total_comments_limit", 10), None, 10 - ) + self.args.current_likes_limit = get_value(getattr(self.args, "total_likes_limit", 300), None, 300) + self.args.current_follow_limit = get_value(getattr(self.args, "total_follows_limit", 50), None, 50) + self.args.current_unfollow_limit = get_value(getattr(self.args, "total_unfollows_limit", 50), None, 50) + self.args.current_comments_limit = get_value(getattr(self.args, "total_comments_limit", 10), None, 10) self.args.current_pm_limit = get_value(getattr(self.args, "total_pm_limit", 10), None, 10) - self.args.current_watch_limit = get_value( - getattr(self.args, "total_watches_limit", 50), None, 50 - ) + self.args.current_watch_limit = get_value(getattr(self.args, "total_watches_limit", 50), None, 50) self.args.current_success_limit = get_value( getattr(self.args, "total_successful_interactions_limit", 100), None, 100 ) - self.args.current_total_limit = get_value( - getattr(self.args, "total_interactions_limit", 1000), None, 1000 - ) - self.args.current_scraped_limit = get_value( - getattr(self.args, "total_scraped_limit", 200), None, 200 - ) - self.args.current_crashes_limit = get_value( - getattr(self.args, "total_crashes_limit", 5), None, 5 - ) + self.args.current_total_limit = get_value(getattr(self.args, "total_interactions_limit", 1000), None, 1000) + self.args.current_scraped_limit = get_value(getattr(self.args, "total_scraped_limit", 200), None, 200) + self.args.current_crashes_limit = get_value(getattr(self.args, "total_crashes_limit", 5), None, 5) def check_limit(self, limit_type=None, output=False): """Returns True if limit reached - else False""" limit_type = SessionState.Limit.ALL if limit_type is None else limit_type # check limits total_likes = self.totalLikes >= int(self.args.current_likes_limit) - total_followed = sum(self.totalFollowed.values()) >= int( - self.args.current_follow_limit - ) + total_followed = sum(self.totalFollowed.values()) >= int(self.args.current_follow_limit) total_unfollowed = self.totalUnfollowed >= int(self.args.current_unfollow_limit) total_comments = self.totalComments >= int(self.args.current_comments_limit) total_pm = self.totalPm >= int(self.args.current_pm_limit) total_watched = self.totalWatched >= int(self.args.current_watch_limit) - total_successful = sum(self.successfulInteractions.values()) >= int( - self.args.current_success_limit - ) - total_interactions = sum(self.totalInteractions.values()) >= int( - self.args.current_total_limit - ) + total_successful = sum(self.successfulInteractions.values()) >= int(self.args.current_success_limit) + total_interactions = sum(self.totalInteractions.values()) >= int(self.args.current_total_limit) - total_scraped = sum(self.totalScraped.values()) >= int( - self.args.current_scraped_limit - ) + total_scraped = sum(self.totalScraped.values()) >= int(self.args.current_scraped_limit) total_crashes = self.totalCrashes >= int(self.args.current_crashes_limit) session_info = [ "Checking session limits:", - f"- Total Likes:\t\t\t\t{'Limit Reached' if total_likes else 'OK'} ({self.totalLikes}/{self.args.current_likes_limit})", - f"- Total Comments:\t\t\t\t{'Limit Reached' if total_comments else 'OK'} ({self.totalComments}/{self.args.current_comments_limit})", + f"- Session Likes Given:\t\t{'Limit Reached' if total_likes else 'OK'} ({self.totalLikes}/{self.args.current_likes_limit})", + f"- Session Comments Given:\t{'Limit Reached' if total_comments else 'OK'} ({self.totalComments}/{self.args.current_comments_limit})", f"- Total PM:\t\t\t\t\t{'Limit Reached' if total_pm else 'OK'} ({self.totalPm}/{self.args.current_pm_limit})", f"- Total Followed:\t\t\t\t{'Limit Reached' if total_followed else 'OK'} ({sum(self.totalFollowed.values())}/{self.args.current_follow_limit})", f"- Total Unfollowed:\t\t\t\t{'Limit Reached' if total_unfollowed else 'OK'} ({self.totalUnfollowed}/{self.args.current_unfollow_limit})", @@ -154,11 +130,16 @@ class SessionState: logger.info(line) return ( - total_likes and getattr(self.args, "end_if_likes_limit_reached", False) - or total_followed and getattr(self.args, "end_if_follows_limit_reached", False) - or total_watched and getattr(self.args, "end_if_watches_limit_reached", False) - or total_comments and getattr(self.args, "end_if_comments_limit_reached", False) - or total_pm and getattr(self.args, "end_if_pm_limit_reached", False), + total_likes + and getattr(self.args, "end_if_likes_limit_reached", False) + or total_followed + and getattr(self.args, "end_if_follows_limit_reached", False) + or total_watched + and getattr(self.args, "end_if_watches_limit_reached", False) + or total_comments + and getattr(self.args, "end_if_comments_limit_reached", False) + or total_pm + and getattr(self.args, "end_if_pm_limit_reached", False), total_unfollowed, total_interactions or total_successful or total_scraped, ) @@ -247,20 +228,20 @@ class SessionState: delta = timedelta(seconds=delta_sec) if not working_hours: return True, 0 - + for n in working_hours: today = current_time.strftime("%Y-%m-%d") # 100% Autonomous: Hybrid Time Format Support (Legacy . vs Modern :) - h_start = n.split('-')[0].replace(":", ".") - h_end = n.split('-')[1].replace(":", ".") - + h_start = n.split("-")[0].replace(":", ".") + h_end = n.split("-")[1].replace(":", ".") + inf_value = f"{h_start} {today}" inf = datetime.strptime(inf_value, "%H.%M %Y-%m-%d") + delta sup_value = f"{h_end} {today}" sup = datetime.strptime(sup_value, "%H.%M %Y-%m-%d") + delta - if sup - inf + timedelta(minutes=1) == timedelta( - days=1 - ) or sup - inf + timedelta(minutes=1) == timedelta(days=0): + if sup - inf + timedelta(minutes=1) == timedelta(days=1) or sup - inf + timedelta(minutes=1) == timedelta( + days=0 + ): logger.debug("Whole day mode.") return True, 0 if time_in_range(inf.time(), sup.time(), current_time.time()): @@ -300,9 +281,7 @@ class SessionStateEncoder(JSONEncoder): return { "id": session_state.id, "total_interactions": sum(session_state.totalInteractions.values()), - "successful_interactions": sum( - session_state.successfulInteractions.values() - ), + "successful_interactions": sum(session_state.successfulInteractions.values()), "total_followed": sum(session_state.totalFollowed.values()), "total_likes": session_state.totalLikes, "total_comments": session_state.totalComments, diff --git a/GramAddict/core/situational_awareness.py b/GramAddict/core/situational_awareness.py index d77c704..318f5cc 100644 --- a/GramAddict/core/situational_awareness.py +++ b/GramAddict/core/situational_awareness.py @@ -10,13 +10,13 @@ After initial learning, 95%+ of situations are handled from memory alone with ZERO LLM calls. This is "Tesla fleet learning" for bots. """ -import logging import hashlib -import time +import logging import re +import time import xml.etree.ElementTree as ET -from typing import Optional, Dict, Any from enum import Enum +from typing import Dict, Optional from GramAddict.core.utils import random_sleep @@ -34,6 +34,7 @@ class SituationType(Enum): class EscapeAction: """Represents a planned escape action.""" + def __init__(self, action_type: str, x: int = 0, y: int = 0, reason: str = "", resource_id: str = ""): self.action_type = action_type # 'click', 'back', 'app_start', 'home_then_app' self.x = x @@ -42,11 +43,19 @@ class EscapeAction: self.resource_id = resource_id def to_dict(self) -> dict: - return {"action_type": self.action_type, "x": self.x, "y": self.y, "reason": self.reason, "resource_id": self.resource_id} + return { + "action_type": self.action_type, + "x": self.x, + "y": self.y, + "reason": self.reason, + "resource_id": self.resource_id, + } @classmethod def from_dict(cls, d: dict) -> "EscapeAction": - return cls(d.get("action_type", "back"), d.get("x", 0), d.get("y", 0), d.get("reason", ""), d.get("resource_id", "")) + return cls( + d.get("action_type", "back"), d.get("x", 0), d.get("y", 0), d.get("reason", ""), d.get("resource_id", "") + ) class SituationEpisodeDB: @@ -56,8 +65,10 @@ class SituationEpisodeDB: Enables instant recall for known situations (0 LLM calls). Stores BOTH positive and negative episodes for full learning. """ + def __init__(self): from GramAddict.core.qdrant_memory import QdrantBase + self._db = QdrantBase("sae_episodes_v1", vector_size=768) def recall(self, situation_signature: str) -> Optional[Dict]: @@ -126,9 +137,31 @@ class SituationEpisodeDB: if not vec: return - # Unique key: situation + action type + success flag - seed = f"{situation_signature}|{action.action_type}|{action.x},{action.y}|{success}" - confidence = 0.8 if success else 0.0 + # Unique key: situation + action type (ignoring success flag for the seed so we update the same entry) + seed = f"{situation_signature}|{action.action_type}|{action.x},{action.y}" + point_id = self._db.generate_uuid(seed) + + current_conf = 0.0 + has_existing = False + try: + points = self._db.client.retrieve( + collection_name=self._db.collection_name, ids=[point_id], with_payload=True, with_vectors=False + ) + if points: + has_existing = True + current_conf = points[0].payload.get("confidence", 0.0) + except Exception: + pass + + if success: + confidence = min(1.0, current_conf + 0.5) if has_existing else 0.8 + else: + confidence = current_conf - 0.5 if has_existing else -0.5 + + if confidence < 0.1 and not success: + self._db.client.delete(collection_name=self._db.collection_name, points_selector=[point_id]) + logger.info("🗑️ [SAE Learn] Action decayed below threshold. Deleted from memory.") + return payload = { "situation": situation_signature[:500], @@ -141,8 +174,10 @@ class SituationEpisodeDB: outcome = "✅ SUCCESS" if success else "❌ FAILURE" self._db.upsert_point( - seed, payload, vector=vec, - log_success=f"🧠 [SAE Learn] {outcome}: '{action.reason}' → Stored for future recall" + seed, + payload, + vector=vec, + log_success=f"🧠 [SAE Learn] {outcome}: '{action.reason}' → Stored for future recall", ) def boost(self, situation_signature: str, action: EscapeAction): @@ -193,7 +228,7 @@ class SituationalAwarenessEngine: try: # Remove XML declaration - clean = re.sub(r'<\?xml.*?\?>', '', xml_dump).strip() + clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip() root = ET.fromstring(clean) except Exception: # If XML is broken, extract what we can with regex @@ -205,17 +240,17 @@ class SituationalAwarenessEngine: packages = set() elements = [] - for elem in root.iter('node'): + for elem in root.iter("node"): a = elem.attrib - pkg = a.get('package', '') + pkg = a.get("package", "") if pkg: packages.add(pkg) - rid = a.get('resource-id', '').strip() - text = a.get('text', '').strip() - desc = a.get('content-desc', '').strip() - bounds = a.get('bounds', '') - clickable = a.get('clickable', 'false') + rid = a.get("resource-id", "").strip() + text = a.get("text", "").strip() + desc = a.get("content-desc", "").strip() + bounds = a.get("bounds", "") + clickable = a.get("clickable", "false") # Only keep nodes with meaningful content if not rid and not text and not desc: @@ -225,10 +260,14 @@ class SituationalAwarenessEngine: if rid: parts.append(f"id={rid.split('/')[-1]}") if text: - parts.append(f"text='{text[:60]}'") + if len(text) > 20: + text = text[:10] + "..." + text[-10:] + parts.append(f"text='{text}'") if desc: - parts.append(f"desc='{desc[:60]}'") - if clickable == 'true': + if len(desc) > 20: + desc = desc[:10] + "..." + desc[-10:] + parts.append(f"desc='{desc}'") + if clickable == "true": parts.append("CLICKABLE") if bounds: parts.append(f"bounds={bounds}") @@ -236,14 +275,14 @@ class SituationalAwarenessEngine: elements.append(" | ".join(parts)) sig = f"PACKAGES: {', '.join(sorted(packages))}\n" - sig += "\n".join(elements[:50]) # Cap at 50 elements + sig += "\n".join(elements[-50:]) # Keep the last 50 elements (highest Z-index/foreground) return sig[:3000] def _compute_situation_hash(self, compressed: str) -> str: """Deterministic hash for situation dedup.""" # Remove volatile parts (timestamps, counters) but keep structural identity - stable = re.sub(r'\d{2}:\d{2}', 'HH:MM', compressed) - stable = re.sub(r'Battery \d+ per cent', 'Battery NN per cent', stable) + stable = re.sub(r"\d{2}:\d{2}", "HH:MM", compressed) + stable = re.sub(r"Battery \d+ per cent", "Battery NN per cent", stable) return hashlib.sha256(stable.encode()).hexdigest()[:32] def perceive(self, xml_dump: str) -> SituationType: @@ -257,12 +296,17 @@ class SituationalAwarenessEngine: xml_lower = xml_dump.lower() blocked_markers = [ - "try again later", "action blocked", "restrict certain activity", - "help us confirm you own", "confirm it's you", - "später erneut versuchen", "bestätige, dass du es bist", - "handlung blockiert", "eingeschränkt", + "try again later", + "action blocked", + "restrict certain activity", + "help us confirm you own", + "confirm it's you", + "später erneut versuchen", + "bestätige, dass du es bist", + "handlung blockiert", + "eingeschränkt", ] - + # Guard: Check if the text matches are relatively isolated (e.g. short strings). # If the string is buried inside a 200-character caption, it's a false positive. # We can regex match text="..." attributes that are less than 60 characters total, @@ -274,18 +318,18 @@ class SituationalAwarenessEngine: return SituationType.DANGER_ACTION_BLOCKED # ── Hardware Guard: Screen Off / Locked ── - if not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True): + if not getattr(self.device.deviceV2, "info", {}).get("screenOn", True): logger.info("📱 [SAE Perceive] Screen is physically OFF.") return SituationType.OBSTACLE_LOCKED_SCREEN # ── System Dialog / Permission Detect (Fast Path) ── packages = set(re.findall(r'package=["\']([^"\']+)["\']', xml_dump)) - app_id = getattr(self.device, 'app_id', 'com.instagram.android') + app_id = getattr(self.device, "app_id", "com.instagram.android") system_dialog_pkgs = { - 'com.google.android.permissioncontroller', - 'com.android.permissioncontroller', - 'com.samsung.android.permissioncontroller' + "com.google.android.permissioncontroller", + "com.android.permissioncontroller", + "com.samsung.android.permissioncontroller", } if any(pkg in system_dialog_pkgs for pkg in packages): logger.info("📱 [SAE Perceive] System permission dialog explicitly detected.") @@ -294,7 +338,7 @@ class SituationalAwarenessEngine: # ── Foreign Environment Detection (package-based) ── # If the main app package is completely absent from the UI hierarchy, # OR if there's a dominant foreign package and no app package, we might have lost the app. - + # If our app is on screen, we trust we are in the app (even if a custom keyboard is open). # We only trigger foreign app classification if our app is completely missing from the screen. is_foreign = False @@ -305,11 +349,11 @@ class SituationalAwarenessEngine: # We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks # for Android System UI variations across different device manufacturers. try: - from GramAddict.core.llm_provider import query_telepathic_llm from GramAddict.core.config import Config - - screen_off = not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True) - + from GramAddict.core.llm_provider import query_telepathic_llm + + screen_off = not getattr(self.device.deviceV2, "info", {}).get("screenOn", True) + prompt = ( "You are a Situation Classifier for a mobile automation agent.\n" "Analyze the given Android UI XML dump. Is this a physical DEVICE_LOCK_SCREEN, " @@ -319,18 +363,27 @@ class SituationalAwarenessEngine: "{\"situation\": \"OBSTACLE_LOCKED_SCREEN\" | \"OBSTACLE_SYSTEM\" | \"OBSTACLE_FOREIGN_APP\"}\n\n" f"XML:\n{self._compress_xml(xml_dump)[:2500]}" ) - + args = {} - try: args = Config().args - except Exception: pass + try: + args = Config().args + except Exception: + pass model = getattr(args, "ai_telepathic_model", "qwen3.5:latest") url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - - res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True) + + res = query_telepathic_llm( + model=model, + url=url, + system_prompt="Strict JSON classifier.", + user_prompt=prompt, + use_local_edge=True, + ) import json + data = json.loads(res) situ_str = data.get("situation", "") - + if situ_str == "OBSTACLE_LOCKED_SCREEN": logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: LOCKED_SCREEN.") return SituationType.OBSTACLE_LOCKED_SCREEN @@ -348,8 +401,9 @@ class SituationalAwarenessEngine: # We explicitly query ScreenMemoryDB. If unknown, we ask the LLM. # This replaces ALL brittle string/ID matching for modals. from GramAddict.core.qdrant_memory import ScreenMemoryDB + screen_memory = ScreenMemoryDB() - + compressed = self._compress_xml(xml_dump) # ── Structural Fast-Check: Content-Creation Overlays ── @@ -358,21 +412,21 @@ class SituationalAwarenessEngine: # and frequently fool the LLM into thinking they are "normal" browsing. # Detecting them structurally is O(1) and requires ZERO LLM calls. creation_flow_markers = ( - 'quick_capture', # Camera / story capture overlay - 'gallery_cancel_button', # Story gallery "Back to Home" button - 'creation_flow', # Post creation wizard - 'reel_camera', # Reel recording interface + "quick_capture", # Camera / story capture overlay + "gallery_cancel_button", # Story gallery "Back to Home" button + "creation_flow", # Post creation wizard + "reel_camera", # Reel recording interface ) - - # Guard: Check against compressed string to ensure these markers ONLY appear - # as resource IDs (e.g. "id=quick_capture_...") and not as plain text in + + # Guard: Check against compressed string to ensure these markers ONLY appear + # as resource IDs (e.g. "id=quick_capture_...") and not as plain text in # user comments/bios (which would look like "text='... creation_flow ...'") - if any(re.search(rf'id=[^\s|]*{marker}', compressed, re.IGNORECASE) for marker in creation_flow_markers): + if any(re.search(rf"id=[^\s|]*{marker}", compressed, re.IGNORECASE) for marker in creation_flow_markers): logger.info("🧠 [SAE Perceive] Content-creation overlay detected structurally → OBSTACLE_MODAL") screen_memory.store_screen(compressed, "OBSTACLE_MODAL") return SituationType.OBSTACLE_MODAL cached_type = screen_memory.get_screen_type(compressed) - + if cached_type: if cached_type == "OBSTACLE_MODAL": return SituationType.OBSTACLE_MODAL @@ -381,9 +435,9 @@ class SituationalAwarenessEngine: # If not cached, query LLM for autonomous structural classification try: - from GramAddict.core.llm_provider import query_telepathic_llm from GramAddict.core.config import Config - + from GramAddict.core.llm_provider import query_telepathic_llm + prompt = ( "You are a Situation Classifier for a mobile automation agent.\n" "Analyze the given Android UI XML dump. Is there a blocking MODAL, DIALOG, or POPUP " @@ -394,21 +448,26 @@ class SituationalAwarenessEngine: "or ANY content-creation flow (reel recording, post editor, live mode) is an OBSTACLE_MODAL — " "it blocks normal navigation and must be dismissed.\n" "Respond ONLY with a valid JSON object strictly matching this schema: " - "{\"situation\": \"OBSTACLE_MODAL\" | \"NORMAL\"}\n\n" + '{"situation": "OBSTACLE_MODAL" | "NORMAL"}\n\n' f"XML:\n{compressed[:2500]}" ) - + args = {} - try: args = Config().args - except Exception: pass + try: + args = Config().args + except Exception: + pass model = getattr(args, "ai_telepathic_model", "qwen3.5:latest") url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - - res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True) + + res = query_telepathic_llm( + model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True + ) import json + data = json.loads(res) situ_str = data.get("situation", "NORMAL") - + if situ_str == "OBSTACLE_MODAL": logger.info("🧠 [Smart Perceive] Screen classified as: OBSTACLE_MODAL.") screen_memory.store_screen(compressed, "OBSTACLE_MODAL") @@ -422,19 +481,28 @@ class SituationalAwarenessEngine: return SituationType.NORMAL + def unlearn_current_state(self, xml_dump: str): + """Purges the current screen's signature from Qdrant to self-heal from hallucinations.""" + compressed = self._compress_xml(xml_dump) + from GramAddict.core.qdrant_memory import ScreenMemoryDB + + screen_memory = ScreenMemoryDB() + screen_memory.purge_screen(compressed) + logger.info("🗑️ [Smart Perceive] Purged cached screen signature to force autonomous re-evaluation.") + # ────────────────────────────────────────────── # 2. PLAN: AI-driven escape strategy # ────────────────────────────────────────────── - - - def _plan_escape_via_llm(self, xml_dump: str, compressed: str, situation_type: SituationType, failed_actions: set = None) -> Optional[EscapeAction]: + def _plan_escape_via_llm( + self, xml_dump: str, compressed: str, situation_type: SituationType, failed_actions: set = None + ) -> Optional[EscapeAction]: """ LLM-powered escape planning for situations where structural scan fails. Called ONLY when recall AND structural planning both miss. """ - from GramAddict.core.llm_provider import query_llm from GramAddict.core.config import Config + from GramAddict.core.llm_provider import query_llm try: args = Config().args @@ -455,29 +523,35 @@ class SituationalAwarenessEngine: "- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n" "- If nothing else works, suggest 'app_start' to force-reopen Instagram\n" "- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n" - "- Return ONLY valid JSON: {\"action\": \"click\"|\"back\"|\"app_start\"|\"unlock\"|\"kill_foreign_apps\"|\"false_positive\", \"x\": N, \"y\": N, \"reason\": \"...\"}" + '- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}' ) - user_prompt = ( - f"Situation type: {situation_type.value}\n\n" - f"Screen content:\n{compressed}\n\n" - ) + user_prompt = f"Situation type: {situation_type.value}\n\n" f"Screen content:\n{compressed}\n\n" if failed_actions: user_prompt += f"Failed actions this session (DO NOT REPEAT): {list(failed_actions)}\n\n" - + user_prompt += "What action should I take to clear this obstacle and return to Instagram? Return JSON only." try: - resp = query_llm(url=url, model=model, prompt=user_prompt, system=system_prompt, - format_json=True, timeout=30, max_tokens=300, temperature=0.0) + resp = query_llm( + url=url, + model=model, + prompt=user_prompt, + system=system_prompt, + format_json=True, + timeout=30, + max_tokens=300, + temperature=0.0, + ) if resp and "response" in resp: import json + data = json.loads(resp["response"]) return EscapeAction( action_type=data.get("action", "back"), x=int(data.get("x", 0)), y=int(data.get("y", 0)), - reason=data.get("reason", "LLM-planned escape") + reason=data.get("reason", "LLM-planned escape"), ) except Exception as e: logger.warning(f"🧠 [SAE] LLM escape planning failed: {e}") @@ -504,24 +578,24 @@ class SituationalAwarenessEngine: logger.info(f"🔓 [SAE Act] Unlocking device: {action.reason}") self.device.unlock() random_sleep(1.0, 2.0) - app_id = getattr(self.device, 'app_id', 'com.instagram.android') + app_id = getattr(self.device, "app_id", "com.instagram.android") self.device.app_start(app_id, use_monkey=True) random_sleep(1.5, 2.5) elif action.action_type == "app_start": logger.info(f"🚀 [SAE Act] Force-starting app: {action.reason}") - app_id = getattr(self.device, 'app_id', 'com.instagram.android') + app_id = getattr(self.device, "app_id", "com.instagram.android") self.device.app_start(app_id, use_monkey=True) random_sleep(2.0, 3.5) elif action.action_type == "kill_foreign_apps": logger.info(f"🔪 [SAE Act] Killing foreign apps: {action.reason}") # The reason string will contain the package name or 'all' - app_id = getattr(self.device, 'app_id', 'com.instagram.android') + app_id = getattr(self.device, "app_id", "com.instagram.android") try: # We can dump current package again, or just get it from device current_pkg = self.device.deviceV2.app_current().get("package") - if current_pkg and current_pkg != app_id and current_pkg not in ('com.android.systemui', 'android'): + if current_pkg and current_pkg != app_id and current_pkg not in ("com.android.systemui", "android"): logger.info(f"🔪 Stopping {current_pkg}") self.device.app_stop(current_pkg) random_sleep(1.0, 2.0) @@ -535,7 +609,7 @@ class SituationalAwarenessEngine: logger.info(f"🏠 [SAE Act] HOME → App Start: {action.reason}") self.device.press("home") random_sleep(0.5, 1.0) - app_id = getattr(self.device, 'app_id', 'com.instagram.android') + app_id = getattr(self.device, "app_id", "com.instagram.android") self.device.app_start(app_id, use_monkey=True) random_sleep(2.0, 3.5) @@ -550,9 +624,10 @@ class SituationalAwarenessEngine: Returns True if an obstacle was successfully cleared, False if already clear or failed. """ from GramAddict.core.exceptions import ActionBlockedError + failed_this_session = set() cleared_something = False - + last_situation = None situation_attempts = 0 @@ -562,9 +637,9 @@ class SituationalAwarenessEngine: xml_dump = initial_xml else: xml_dump = self.device.dump_hierarchy() - + situation = self.perceive(xml_dump) - + if last_situation != situation: situation_attempts = 0 last_situation = situation @@ -582,13 +657,11 @@ class SituationalAwarenessEngine: logger.error("🚫 [SAE CRITICAL] Instagram Action Block detected! Halting to protect account.") raise ActionBlockedError("Instagram action block detected by SAE.") - logger.warning( - f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})" - ) + logger.warning(f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})") # ── COMPRESS for memory lookup ── compressed = self._compress_xml(xml_dump) - + # ── RECALL from memory ── recalled = self.episodes.recall(compressed) if recalled: @@ -597,7 +670,7 @@ class SituationalAwarenessEngine: action = EscapeAction.from_dict(recalled) logger.info(f"🧠 [SAE] Using recalled strategy: {action.reason}") else: - logger.info(f"🧠 [SAE] Recalled strategy already failed this session. Using LLM planning.") + logger.info("🧠 [SAE] Recalled strategy already failed this session. Using LLM planning.") recalled = None if not recalled: @@ -605,14 +678,23 @@ class SituationalAwarenessEngine: logger.info("🧠 [SAE] Autonomous Blank Start: Escalating to LLM-assisted escape planning...") action = self._plan_escape_via_llm(xml_dump, compressed, situation, failed_this_session) elif situation_attempts == 3: - action = EscapeAction("app_start", reason=f"Escalation level 4: force app restart after {situation_attempts} failed attempts on this situation") + action = EscapeAction( + "app_start", + reason=f"Escalation level 4: force app restart after {situation_attempts} failed attempts on this situation", + ) else: - action = EscapeAction("home_then_app", reason=f"Nuclear escalation: HOME + app_start after {situation_attempts} failed attempts on this situation") + action = EscapeAction( + "home_then_app", + reason=f"Nuclear escalation: HOME + app_start after {situation_attempts} failed attempts on this situation", + ) # ── EXECUTE ── if action.action_type == "false_positive": - logger.warning(f"🧠 [SAE Unlearn] LLM identified false positive obstacle. Overwriting Qdrant memory to NORMAL.") + logger.warning( + "🧠 [SAE Unlearn] LLM identified false positive obstacle. Overwriting Qdrant memory to NORMAL." + ) from GramAddict.core.qdrant_memory import ScreenMemoryDB + ScreenMemoryDB().store_screen(compressed, "NORMAL") self._consecutive_failures = 0 return True @@ -623,8 +705,8 @@ class SituationalAwarenessEngine: # ── VERIFY ── post_xml = self.device.dump_hierarchy() post_situation = self.perceive(post_xml) - reached_normal = (post_situation == SituationType.NORMAL) - situation_changed = (post_situation != situation) + reached_normal = post_situation == SituationType.NORMAL + situation_changed = post_situation != situation if reached_normal: # ── LEARN FULL SUCCESS ── @@ -635,7 +717,9 @@ class SituationalAwarenessEngine: elif situation_changed: # ── LEARN PARTIAL SUCCESS ── self.episodes.learn(compressed, action, True) - logger.info(f"🔄 [SAE] Situation changed from {situation.value} to {post_situation.value}. Continuing recovery...") + logger.info( + f"🔄 [SAE] Situation changed from {situation.value} to {post_situation.value}. Continuing recovery..." + ) # We do not increment consecutive_failures or situation_attempts because we made progress # The next loop iteration will clear failed_this_session since last_situation != situation else: diff --git a/GramAddict/core/telepathic_engine.py b/GramAddict/core/telepathic_engine.py index 0b3f644..bbeefb4 100644 --- a/GramAddict/core/telepathic_engine.py +++ b/GramAddict/core/telepathic_engine.py @@ -1,91 +1,29 @@ -import base64 -import json import logging -import math -import os -import re -import time -import xml.etree.ElementTree as ET -from typing import Dict, Optional +from typing import Optional -from colorama import Fore - -from GramAddict.core.diagnostic_dump import dump_ui_state -from GramAddict.core.llm_provider import query_telepathic_llm -from GramAddict.core.qdrant_memory import QdrantBase +from GramAddict.core.perception.action_memory import ActionMemory +from GramAddict.core.perception.intent_resolver import IntentResolver +from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator +from GramAddict.core.perception.spatial_parser import SpatialNode, SpatialParser logger = logging.getLogger(__name__) -# ── Screen Zone Constants (fraction of screen height) ── -# Used for positional sanity checking instead of hardcoded resource-IDs. -STATUS_BAR_ZONE = 0.10 # Top 10% = Android status bar (increased to prevent hallucinations) -NAV_BAR_ZONE = 0.90 # Bottom 10% = Android nav bar (consistent with tab enforcement) -MAX_BUTTON_AREA = 150000 # Buttons/icons should be smaller than this (px²) -MAX_CONTAINER_AREA = 500000 # Anything above this is a full-screen container - -# ── Nav Intent Keywords (SSOT — used by all structural guards) ── -# -# TWO purpose-specific sets prevent the Guard Civil War: -# -# NAV_ZONE_BYPASS_KEYWORDS: Intents ALLOWED to target elements in the -# bottom nav bar zone. Broad — includes tabs, DMs, profile stats. -# -# NAV_TAB_KEYWORDS: Intents that REQUIRE their target to be at the bottom. -# Narrow — only actual navigation tabs. "following"/"follower" are profile -# stats at the TOP of the screen, not tabs. - -NAV_ZONE_BYPASS_KEYWORDS = [ - "tab", - "navigation", - "reels tab", - "profile tab", - "home tab", - "explore tab", - "message tab", - "direct message", - "inbox", - "dm", - "notification", - "heart icon", - "following", - "follower", - "followers", -] - -NAV_TAB_KEYWORDS = [ - "reels tab", - "profile tab", - "home tab", - "explore tab", - "message tab", -] - -# Cache files -MEMORY_FILE = "telepathic_memory.json" -BLACKLIST_FILE = "telepathic_blacklist.json" - class TelepathicEngine: """ - The Self-Learning Telepathic UI Engine + The Spatial Perception Engine (Facade) - Completely replaces static Locators (XPath/Regex). - Transforms UI Nodes into natural language semantics, generates vector embeddings, - and returns the node mathematically closest to the target intent. - - Philosophy: ZERO hardcoded Instagram IDs. - Instead of maintaining brittle ID lists, the engine uses: - 1. Structural heuristics (size, position, element class) — app-agnostic - 2. Post-click verification — caller confirms if the click worked - 3. Negative learning — failed clicks are blacklisted and never repeated - 4. Positive reinforcement — confirmed clicks are cached for instant recall - - The engine never trusts a VLM output blindly. It returns candidates, - and the caller uses `confirm_click()` or `reject_click()` to teach it. + Replaces the legacy 2100+ LoC Regex/XML engine with a structural Topo-Graph approach. + Delegates all responsibilities to SOLID, testable sub-modules. """ _instance = None - _last_click_context: Optional[dict] = None # Tracks what we last returned for feedback + + # Delegate Modules + _parser: SpatialParser + _resolver: IntentResolver + _memory: ActionMemory + _evaluator: SemanticEvaluator @classmethod def get_instance(cls): @@ -95,2055 +33,268 @@ class TelepathicEngine: @classmethod def reset(cls): - """Reset the singleton instance (useful for tests).""" cls._instance = None - cls._last_click_context = None - - def wipe(self): - """Perform a full 'Blank Start' wipe of all AI caches and persistent memories.""" - logger.warning("🔥 [TelepathicEngine] Wiping all AI caches and persistent records for BLANK START.") - - # 1. Clear JSON files - for f in [MEMORY_FILE, BLACKLIST_FILE]: - if os.path.exists(f): - try: - os.remove(f) - logger.info(f"🗑️ Deleted persistent file: {f}") - except Exception as e: - logger.error(f"❌ Failed to delete {f}: {e}") - - # 2. Clear Qdrant collections - try: - if hasattr(self, "embedding_helper") and self.embedding_helper: - self.embedding_helper.wipe_collection() - except Exception as e: - logger.warning(f"⚠️ Could not wipe Qdrant collection (likely doesn't exist): {e}") - - try: - if hasattr(self, "ui_memory") and self.ui_memory: - self.ui_memory.wipe_collection() - except Exception as e: - logger.warning(f"⚠️ Could not wipe UIMemoryDB collection: {e}") - - # 3. Clear In-memory state - self._memory = {} - self._blacklist = {} - self._embedding_cache = {} - self._intent_cache = {} def __init__(self): - from GramAddict.core.qdrant_memory import UIMemoryDB + self._parser = SpatialParser() + self._resolver = IntentResolver() + self._memory = ActionMemory() + self._evaluator = SemanticEvaluator() - self.ui_memory = UIMemoryDB() - self.embedding_helper = QdrantBase("telepathic_engine_cache") - self._embedding_cache: Dict[str, list] = {} - self._intent_cache: Dict[str, list] = {} - # Load blacklist (negative learnings) into memory - self._blacklist = self._load_json(BLACKLIST_FILE) - # Load positive cache - self._memory = self._load_json(MEMORY_FILE) - - # ────────────────────────────────────────────── - # Core Math - # ────────────────────────────────────────────── - - def _cosine_similarity(self, v1: list, v2: list) -> float: - if not v1 or not v2 or len(v1) != len(v2): - return 0.0 - dot_product = sum(a * b for a, b in zip(v1, v2)) - magnitude_v1 = math.sqrt(sum(a * a for a in v1)) - magnitude_v2 = math.sqrt(sum(b * b for b in v2)) - if magnitude_v1 == 0 or magnitude_v2 == 0: - return 0.0 - return dot_product / (magnitude_v1 * magnitude_v2) - - def _get_cached_embedding(self, text: str, is_intent: bool = False) -> Optional[list]: - cache = self._intent_cache if is_intent else self._embedding_cache - if text in cache: - return cache[text] - if len(self._embedding_cache) > 2000: - self._embedding_cache.clear() - vec = self.embedding_helper._get_embedding(text) - if vec: - cache[text] = vec - return vec - - # ────────────────────────────────────────────── - # Persistent JSON helpers - # ────────────────────────────────────────────── - - @staticmethod - def _load_json(path: str) -> dict: + def wipe(self): + """Wipe all persistent state for Blank Start.""" + logger.warning("🔥 [TelepathicEngine] Wiping state (delegating to ActionMemory layer).") try: - if os.path.exists(path): - with open(path, "r") as f: - return json.load(f) - except Exception: - pass - return {} - - @staticmethod - def _save_json(path: str, data: dict): - try: - with open(path, "w") as f: - json.dump(data, f, indent=4) + self._memory.ui_memory.wipe_collection() except Exception as e: - logger.warning(f"Could not save {path}: {e}") + logger.warning(f"Could not wipe Qdrant collection: {e}") # ────────────────────────────────────────────── - # XML Parsing + # Core Resolution Engine # ────────────────────────────────────────────── - def _extract_semantic_nodes(self, xml_string: str, intent: str = None, threshold: float = 0.0) -> list[dict]: + def find_best_node(self, xml_string: str, intent_description: str, device=None, **kwargs) -> Optional[dict]: """ - Parses Android UI XML and extracts clickable/interactive nodes. - If intent and threshold are provided, it filters nodes by semantic score. + Public facade for resolving a node. + Translates Android UI bounds into standard GramAddict node dicts. """ - nodes = [] - try: - clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_string).strip() - root = ET.fromstring(clean_xml) + logger.debug(f"🧠 [SpatialEngine] Resolving intent: '{intent_description}'") - for elem in root.iter("node"): - attrib = elem.attrib - text = attrib.get("text", "").strip() - content_desc = attrib.get("content-desc", "").strip() - res_id = attrib.get("resource-id", "").strip() - class_name = attrib.get("class", "").strip() + # 0. DM Thread Guard: Block profile intents inside DM threads + is_dm_thread = "direct_thread_header" in xml_string or "row_thread_composer_edittext" in xml_string + if is_dm_thread: + profile_keywords = ["profile", "follow", "first image", "grid", "avatar", "story ring", "feed"] + if any(k in intent_description.lower() for k in profile_keywords): + logger.warning(f"🛡️ [DM Guard] Blocked profile/feed intent '{intent_description}' inside DM thread.") + return {"blocked_by_dm_thread": True} - clickable = attrib.get("clickable", "false") == "true" - scrollable = attrib.get("scrollable", "false") == "true" - long_clickable = attrib.get("long-clickable", "false") == "true" + # 0.5 Comments Disabled Guard + if "comment" in intent_description.lower(): + if "comments are turned off" in xml_string.lower(): + logger.warning("🛡️ [Comment Guard] Comments are disabled on this post.") + return {"skip": True, "semantic": "comments disabled"} - semantic_res = res_id and any(x in res_id.lower() for x in ["button", "tab", "icon", "action", "menu"]) - has_semantic_weight = bool(content_desc or semantic_res) + # 1.25 Grid Fast-Path (Deterministically bypass VLM for first grid item) + if "first image in explore grid" in intent_description.lower(): + nodes_dicts = self._extract_semantic_nodes(xml_string) + fast_node = self._grid_fast_path(intent_description, nodes_dicts, kwargs.get("skip_positions")) + if fast_node: + return fast_node - if not (clickable or scrollable or long_clickable or has_semantic_weight): - continue + # 1. Parse into Spatial Topology + root = self._parser.parse(xml_string) + if not root: + logger.warning("Failed to parse Spatial Topology.") + return None - if not text and not content_desc and not res_id: - continue + # 2. Extract interactable candidates + candidates = self._parser.get_clickable_nodes(root) - desc_parts = [] - if text: - desc_parts.append(f"text: '{text}'") - if content_desc: - desc_parts.append(f"description: '{content_desc}'") - if res_id: - clean_id = res_id.split("/")[-1].replace("_", " ") - desc_parts.append(f"id context: '{clean_id}'") + # 3. Resolve intent against candidates + best_node = self._resolver.resolve(intent_description, candidates) - semantic_string = ", ".join(desc_parts) - if not semantic_string: - continue + if not best_node: + logger.warning(f"No viable nodes found for intent: '{intent_description}'") + return None - bounds_str = attrib.get("bounds", "") - match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str) - if not match: - continue - - left, top, right, bottom = map(int, match.groups()) - center_x = (left + right) // 2 - center_y = (top + bottom) // 2 - width = right - left - height = bottom - top - area = width * height - - node = { - "semantic_string": semantic_string, - "x": center_x, - "y": center_y, - "width": width, - "height": height, - "area": area, - "raw_bounds": bounds_str, - "resource_id": res_id, - "class_name": class_name, - "naf": attrib.get("NAF", "false").lower() == "true", - "selected": attrib.get("selected", "false").lower() == "true", - "text": text, - "content_desc": content_desc, - "original_attribs": {"text": text, "desc": content_desc}, - } - - # Apply structural filter before scoring if intent is provided - if intent: - # We use a default screen_height if we call extract directly with intent - if not self._structural_sanity_check(node, intent, screen_height=2400): - continue - - # Compute score for filtering - score = self._compute_quick_score(intent, semantic_string) - if score < threshold: - continue - node["score"] = score - - nodes.append(node) - except Exception as e: - logger.error(f"Telepathic XML parsing failed: {e}") - - return nodes - - def _compute_quick_score(self, intent: str, semantic: str) -> float: - """Helper for legacy _extract_semantic_nodes filtering.""" - clean_intent = re.sub(r"[^\w\s]", " ", intent.lower().replace("_", " ")) - clean_semantic = re.sub(r"[^\w\s]", " ", semantic.lower().replace("_", " ")) - intent_words = set(clean_intent.split()) - semantic_words = set(clean_semantic.split()) - common = intent_words.intersection(semantic_words) - return len(common) / len(intent_words) if intent_words else 0.0 - - # ────────────────────────────────────────────── - # Forbidden Action Guard (NEVER click these) - # ────────────────────────────────────────────── - # These are elements the bot must NEVER interact with, regardless of - # what the VLM or any heuristic says. They trigger side-effectful actions - # like creating content, going live, modifying account, etc. - # Pattern-matched against text, content-desc, AND resource-id (lowered). - FORBIDDEN_ACTIONS = [ - # ── Content Creation (absolute red line) ── - "add to story", - "create story", - "your story", - "neue story", - "story erstellen", - "go live", - "live gehen", - "create post", - "beitrag erstellen", - "neuer beitrag", - "create reel", - "reel erstellen", - "camera", - "kamera", - # ── Account Modification ── - "edit profile", - "profil bearbeiten", - "switch account", - "konto wechseln", - "log out", - "abmelden", - "ausloggen", - "delete account", - "konto löschen", - # ── Dangerous Social Actions ── - "close friend", - "enge freunde", - "block", - "blockieren", - "report", - "melden", - "restrict", - "einschränken", - # ── Shopping/Payment ── - "checkout", - "buy now", - "purchase", - "jetzt kaufen", - "zur kasse", - "bezahlen", - ] - # Resource-ID fragments that indicate forbidden elements - FORBIDDEN_ID_FRAGMENTS = [ - "story_create", - "story_camera", - "reel_camera", - "camera_button", - "creation_tab", - "live_button", - "go_live", - "close_friend", - "close_friends", - "reel_empty_badge", # This is the "Add to story" badge that caused the real-world bug - "quick_capture", # Camera/story capture overlay — absolute navigation trap - ] - - def _is_forbidden_action(self, node: dict) -> bool: - """ - Returns True if this node represents a FORBIDDEN action the bot must never trigger. - Checks text, content-desc, and resource-id against the forbidden lists. - This is the LAST LINE OF DEFENSE against hallucinated VLM targets. - """ - text = node.get("text", "").lower() - desc = node.get("description", "").lower() - res_id = node.get("resource_id", "").lower() - semantic = node.get("semantic_string", "").lower() - - searchable = f"{text} {desc} {res_id} {semantic}" - - for forbidden in self.FORBIDDEN_ACTIONS: - if forbidden in searchable: - return True - - for frag in self.FORBIDDEN_ID_FRAGMENTS: - if frag in res_id: - return True - - return False - - # ────────────────────────────────────────────── - # Structural Sanity (app-agnostic, no hardcoded IDs) - # ────────────────────────────────────────────── - - def _structural_sanity_check(self, node: dict, intent_description: str, screen_height: int = 2400) -> bool: - """ - App-agnostic structural validation. Checks physical properties - (size, position, element class) — NOT resource-ID strings. - - Returns False if the node is structurally implausible as a click target. - """ - import numbers - - if not isinstance(screen_height, numbers.Number): - screen_height = 2400 - else: - try: - screen_height = int(screen_height) - except (ValueError, TypeError): - screen_height = 2400 - # 1. Reject massive containers (full-screen views, recycler views) - # UNLESS the intent explicitly targets media - low_intent = intent_description.lower() - is_media_intent = any(k in low_intent for k in ["video", "photo", "reel", "media", "post"]) - is_grid_item_intent = any(k in low_intent for k in ["grid", "list", "first", "item", "row"]) - - # If the intent specifically mentions looking for an item within a grid/list, - # we must block massive parent containers even if the word 'post', 'photo' or 'video' is present. - if is_grid_item_intent: - is_media_intent = False - - if node.get("area", 0) > MAX_CONTAINER_AREA and not is_media_intent: - return False - - # 2. Reject nodes in the Android status bar zone (top 10%) - # UNLESS they are explicitly known top-bar interaction elements - top_safe_ids = ["row_feed_photo_profile_name", "media_header_user", "action_bar", "secondary_label"] - res_id = node.get("resource_id", "").lower() - top_bypass = any(k in res_id for k in top_safe_ids) - - if node.get("y", 0) < screen_height * STATUS_BAR_ZONE and not top_bypass: - return False - - # 3. Reject nodes in the Navigation Bar zone (bottom 6% - adjusted for accuracy) - # UNLESS the intent is explicitly about navigation tabs, profile stats, OR popup modals - nav_keywords = NAV_ZONE_BYPASS_KEYWORDS - modal_keywords = ["dismiss", "ok", "cancel", "accept", "allow", "deny", "action", "obstacle", "popup"] - - low_intent = intent_description.lower() - is_nav_intent = any(k in low_intent for k in nav_keywords) - is_modal_intent = any(k in low_intent for k in modal_keywords) - - # Resource-ID bypass for profile header elements that sit low - safe_ids = ["following", "follower", "post_count", "button_edit_profile", "button_share_profile", "direct_tab"] - res_id = node.get("resource_id", "").lower() - id_bypass = any(k in res_id for k in safe_ids) - - threshold = screen_height * NAV_BAR_ZONE - - if node.get("y", 0) > threshold and not (is_nav_intent or is_modal_intent or id_bypass): - # Not an AI error—this is the deterministic prep-filter culling the NavBar before VLM logic. - return False - - # STRICT NAVIGATION TAB ENFORCEMENT - # Navigation tabs MUST be at the bottom (>= 0.92 of screen height). - # We reject any navigation tab hallucinated in the middle of the screen. - # STRICT TAB ENFORCEMENT: Only actual tab intents must be at the bottom. - is_tab_intent = any(k in low_intent for k in NAV_TAB_KEYWORDS) - if is_tab_intent and node.get("y", 0) < screen_height * 0.92: - # Silently filter non-bottom elements for tab intents to prevent log spam - return False - - # STRICT STORY TRAY ENFORCEMENT - # Story rings MUST be at the top of the screen (y < 30% of screen height). - # We reject any story ring hallucinated in the feed (e.g. post headers). - is_story_intent = "story ring" in low_intent or "story avatar" in low_intent - if is_story_intent and node.get("y", 0) > screen_height * 0.30: - return False - - # 3.5. Reject Action Bars for Content Intents - # If we are looking for user content (posts, grid items, media), never click an action bar or tab bar. - is_content_intent = any(k in low_intent for k in ["post", "grid", "media", "photo", "video", "reel", "item"]) - res_id_lower = node.get("resource_id", "").lower() - is_generic_bar = any(k in res_id_lower for k in ["action_bar", "tab_bar", "navigation_bar", "tab_layout"]) - if is_content_intent and is_generic_bar: - return False - - # 4. Reject own profile/story if the intent is not explicitly looking for it. - # Intuitively, "profile picture avatar story ring" means "click a user's story". - # If we are looking for a story/profile, we must NOT click our OWN story. - is_targeting_own_profile = any(k in low_intent for k in ["own profile", "my profile", "own story"]) - if not is_targeting_own_profile: - semantic_lower = node.get("semantic_string", "").lower() - if "your story" in semantic_lower: - return False - - bot_username = self._get_current_username() - if bot_username and bot_username in semantic_lower: - # Rejecting bot's own username to prevent clicking itself - return False - - # STRICT BUTTON GUARD - # If the intent explicitly asks for a button (e.g. "like button"), - # we reject nodes that are explicitly user profile links or text descriptions - # that contain the word 'profile' or 'go to', UNLESS the intent also asks to go to a profile. - is_button_intent = "button" in low_intent - is_profile_intent = "profile" in low_intent or "user" in low_intent - - if is_button_intent and not is_profile_intent: - semantic_lower = node.get("semantic_string", "").lower() - if "profile" in semantic_lower or "go to" in semantic_lower: - return False - # 5. Language-Agnostic Modal/Menu Guard - # Prevent clicks on items inside dialogs, bottom sheets, or context menus - # UNLESS the intent explicitly targets a menu or modal interaction. - # This completely nullifies the risk of accidentally clicking "Favorites" or "Unfollow" from a dropdown. - menu_id_indicators = ["menu_item", "option", "bottom_sheet", "dialog", "action_sheet", "popup"] - is_menu_node = any(m in node.get("resource_id", "").lower() for m in menu_id_indicators) - - # If the intent doesn't sound like it wants a menu... - wants_menu_intent = any(k in low_intent for k in ["menu", "option", "more", "dismiss", "cancel", "modal"]) - if is_menu_node and not wants_menu_intent: - return False - - # 6. Reject nodes with zero area (invisible) - if node.get("area", 0) == 0: - return False - - # 7. FORBIDDEN ACTION GUARD - # The bot must NEVER click elements that create content, modify the account, - # or trigger irreversible actions. This is the pre-filter level — - # VLM will never even see these nodes. - if self._is_forbidden_action(node): - logger.debug( - f"🛡️ [Forbidden Guard] Blocking dangerous element for intent '{intent_description}': " - f"{node.get('semantic_string', 'N/A')}" + # 3.5 Following Button Guard + if "follow" in intent_description.lower() and "unfollow" not in intent_description.lower(): + semantic = ( + (best_node.text or "") + " " + (best_node.content_desc or "") + " " + (best_node.resource_id or "") ) - return False + semantic = semantic.lower() + if "following" in semantic or "gefolgt" in semantic or "requested" in semantic or "angefragt" in semantic: + return {"skip": True, "semantic": "already_followed"} - return True + # 4. Track action + self._memory.track_click(intent_description, best_node, xml_string) - def _is_blacklisted(self, intent: str, semantic_string: str) -> bool: - """Checks if this intent→node mapping was previously rejected via negative learning.""" - blacklisted = self._blacklist.get(intent, []) - return semantic_string in blacklisted + # Translate to old GramAddict dict format for backward compatibility + return self._translate_node(best_node) - # ────────────────────────────────────────────── - # App Context Guard - # ────────────────────────────────────────────── - - def _get_current_username(self) -> str: - """Helper to get the current IG username from config, safely.""" - username = getattr(self, "_cached_username", None) - if not username: - try: - from GramAddict.core.config import Config - - cfg = Config() - raw_u = cfg.args.username if hasattr(cfg, "args") and hasattr(cfg.args, "username") else None - if raw_u and isinstance(raw_u, list) and len(raw_u) > 0: - username = str(raw_u[0]).lower() - elif isinstance(raw_u, str): - username = raw_u.lower() - else: - username = "" - except Exception: - username = "" - self._cached_username = username - return username - - # ────────────────────────────────────────────── - - def _is_instagram_context(self, nodes: list[dict]) -> bool: - """ - Returns True only if the extracted nodes appear to come from the target app. - Checks for the presence of the dynamic app_id in resource-ID prefixes. - """ - app_id = getattr(self, "_cached_app_id", None) - if not app_id: - from GramAddict.core.config import Config - - try: - cfg = Config() - app_id = ( - cfg.args.app_id if hasattr(cfg, "args") and hasattr(cfg.args, "app_id") else "com.instagram.android" - ) - except Exception: - app_id = "com.instagram.android" - self._cached_app_id = app_id - - for n in nodes: - rid = n.get("resource_id", "") - if app_id in rid: - return True - return False - - # ────────────────────────────────────────────── - # Stage 0: Deterministic Keyword Fast Path - # ────────────────────────────────────────────── - - def _keyword_match_score(self, intent_description: str, nodes: list[dict]) -> Optional[dict]: - """ - Pure string-matching stage. Extracts keywords from the intent and - matches them against node text, description, and resource-id. - Returns the best matching node as a result dict, or None. - - This eliminates ~90% of embedding/VLM calls for common UI intents. - ZERO AI cost — runs entirely on CPU string ops. - """ - # Extract meaningful keywords from intent (strip common filler words) - filler = {"tap", "the", "button", "on", "in", "a", "an", "of", "for", "to", "and", "or", "input", "text", "box"} - intent_words = set( - w.lower() for w in re.split(r"\W+", intent_description) if w and w.lower() not in filler and len(w) > 1 - ) - - if not intent_words: - return None - - # Check for absolute trap/anomaly dismissal intent to bypass LLM parsing - intent_lower = intent_description.lower() - if "close" in intent_lower or "dismiss" in intent_lower or "escape" in intent_lower: - priority_keywords = [ - "done", - "close", - "cancel", - "schließen", - "abbrechen", - "dismiss", - "not now", - "clear text", - ] - best_match = None - best_priority = 999 - for n in nodes: - s = n.get("semantic_string", "").lower() + " " + n.get("resource_id", "").lower() - for i, k in enumerate(priority_keywords): - if k in s and i < best_priority: - best_priority = i - best_match = n - - if best_match: - logger.info( - f"⏭️ [Keyword Fast Path] Detected semantic escape hatch: '{best_match.get('semantic_string')}'" - ) - return { - "x": best_match["x"], - "y": best_match["y"], - "score": 1.0, - "semantic": best_match.get("semantic_string", "Escape Hatch"), - "source": "keyword_fast_path", - "original_attribs": best_match.get("original_attribs", {}), - } - - # Expand known Instagram aliases to avoid sending UI basics to the LLM mappings - aliases = { - "reels": ["clips", "reel"], - "explore": ["search"], - "home": ["main"], - "like": ["heart"], - "comment": ["reply"], - "profile": ["user", "account"], - # Structural feed content aliases for autonomous extraction - "author": ["name", "profile_name", "owner"], - "username": ["name"], - "header": ["header"], - "post": ["feed", "photo", "carousel", "clips"], - "media": ["imageview", "video", "clips", "media_group", "carousel"], - "image": ["imageview"], - "content": ["media", "group", "imageview", "video", "caption"], - "description": ["content-desc", "caption"], - } - - scored = [] - for node in nodes: - sem = node.get("semantic_string", "").lower() - rid = node.get("resource_id", "").lower().replace("_", " ").replace("/", " ") - desc_text = node.get("original_attribs", {}).get("desc", "").lower() - node_text = node.get("original_attribs", {}).get("text", "").lower() - - # Combine all searchable fields - searchable = f"{sem} {rid} {desc_text} {node_text}" - - # Count how many intent keywords appear in the node's text (including aliases) - # Using WORD BOUNDARY matching to prevent 'follow' → '5.107following' false positives - hits = 0 - for w in intent_words: - # Word-boundary match: 'follow' must NOT match 'following' or '5107following' - if re.search(r"(?:^|[\s,._\-:])" + re.escape(w) + r"(?:$|[\s,._\-:!?])", searchable): - hits += 1 - elif w in aliases: - for alias in aliases[w]: - if re.search(r"(?:^|[\s,._\-:])" + re.escape(alias) + r"(?:$|[\s,._\-:!?])", searchable): - hits += 1 - break - - if hits == 0: - continue - - # Score = ratio of intent keywords matched - score = hits / len(intent_words) - - # ── Brevity Bonus ── - # Give a slight boost (up to 0.2) if the matched words make up a large portion of the node's text. - # This ensures "Profile" scores higher than "Profile picture of user" for the intent "open profile". - node_words = len([w for w in searchable.split() if w.strip()]) - if node_words > 0: - score += (hits / node_words) * 0.2 - - # ── Anti-confusion: reject stat counters for action intents ── - # 'follow button' intent should NOT match '5.107following' stat counter - if "follow" in intent_words or "like" in intent_words: - if re.search(r"\d+\s*(follow|like|post|comment)", searchable): - # This is a stat counter (e.g., '2.361 followers'), not an action button - score *= 0.3 # Heavy penalty - - # Thresholding: - # - Tab intents (actual bottom tabs): Require 100% exact match to avoid feed cross-talk - # - Non-tab navigation (following list, DM, etc.): Standard thresholds - # - Short intents (1-2 words): Require at least 50% hit (0.45) - # - Longer intents: Require 75% to avoid false matches on noisy screens. - is_tab_intent = any(k in intent_lower for k in NAV_TAB_KEYWORDS) - if is_tab_intent: - threshold = 1.0 - else: - threshold = 0.45 if len(intent_words) <= 2 else 0.75 - - if score >= threshold: - scored.append((node, score)) - - if not scored: - return None - - # Sort by score desc, then by area asc (prefer smallest/most atomic) - scored.sort(key=lambda x: (-x[1], x[0].get("area", 999999))) - best_node, best_score = scored[0] - - # Check for already-liked state - if "like" in intent_description.lower(): - desc = best_node.get("original_attribs", {}).get("desc", "").lower() - text = best_node.get("original_attribs", {}).get("text", "").lower() - if "liked" in desc or "liked" in text: - logger.info("⏭️ [Keyword Fast Path] Post is already Liked. Skipping.") - return {"x": None, "y": None, "score": 1.0, "semantic": "already_liked", "skip": True} - - # Check for already-followed state - if "follow" in intent_description.lower(): - desc = best_node.get("original_attribs", {}).get("desc", "").lower() - text = best_node.get("original_attribs", {}).get("text", "").lower() - if re.search(r"\b(following|requested|folgst du|angefragt|gefolgt)\b", desc + " " + text): - logger.info("⏭️ [Keyword Fast Path] User is already Followed. Skipping.") - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - - logger.info( - f"⚡ [Keyword Fast Path] Instant match for '{intent_description}' → {best_node['semantic_string']} (KeyScore: {best_score:.2f})" - ) - self._track_click(intent_description, best_node) + def _translate_node(self, node: SpatialNode) -> dict: + """Adapts the new SpatialNode to the legacy GramAddict dictionary format.""" return { - "x": best_node["x"], - "y": best_node["y"], - "score": 0.95, # High confidence — deterministic match - "semantic": best_node["semantic_string"], - "area": best_node.get("area", 0), - "source": "keyword", - "original_attribs": best_node.get("original_attribs", {}), + "x": node.center_x, + "y": node.center_y, + "bounds": f"[{node.x1},{node.y1}][{node.x2},{node.y2}]", + "text": node.text, + "description": node.content_desc, + "id": node.resource_id, + "class": node.class_name, + "original_attribs": node.to_dict(), } - # ────────────────────────────────────────────── - # Core: Find Best Node - # ────────────────────────────────────────────── - - def find_best_node( - self, xml_hierarchy: str, intent_description: str, min_confidence: float = 0.82, device=None, **kwargs - ) -> Optional[dict]: + def _extract_semantic_nodes( + self, xml_string: str, semantic_query: str = None, threshold: float = 0.7, **kwargs + ) -> list: """ - Wrapped find_best_node that runs the VLM semantic trap door guard on positive matches. - """ - res = self._find_best_node_inner(xml_hierarchy, intent_description, min_confidence, device, **kwargs) - - if res and not res.get("skip") and res.get("x") is not None: - # Trap Guard for highly destructive intents - low_intent = intent_description.lower() - if any(k in low_intent for k in ["like", "follow", "comment"]): - if not self._vlm_trap_guard(intent_description, res, device): - logger.error( - f"🚨 [VLM TRAP GUARD] Aborting action for '{intent_description}'. Semantic trap door detected." - ) - return None - return res - - def _vlm_trap_guard(self, intent: str, resolved_node: dict, device) -> bool: - """ - Ultimate sanity check mapping semantic resolution back to pixels using VLM. - """ - if not device: - return True - try: - from GramAddict.core.config import Config - - args = getattr(Config(), "args", None) - use_vision = getattr(args, "ai_vision_navigation", False) if args else False - if not use_vision: - return True - - logger.info("🛡️ [Sanity Guard] Performing semantic VLM trap check...") - screenshot_b64 = device.get_screenshot_b64() - sys_prompt = ( - "You are an AI Security Sentinel for an Instagram automation tool. " - "The bot is about to click an element based on text similarity, but Instagram sets 'Trap Doors' " - "where invisible buttons have fake content-desc like 'Like'. " - "Does the provided semantic element TRULY look like the intent? " - 'Output JSON only: {"safe": boolean, "reason": "string"}' - ) - user_prompt = ( - f"Intent: {intent}\nNode Semantic: {resolved_node.get('semantic', '')}\nIs this safe or a trap?" - ) - - from GramAddict.core.llm_provider import extract_json, query_llm - - model = getattr(args, "ai_telepathic_model", "llama3.2-vision") if args else "llama3.2-vision" - url = ( - getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - if args - else "http://localhost:11434/api/generate" - ) - - resp = query_llm( - url, - model, - user_prompt, - system=sys_prompt, - images_b64=[screenshot_b64], - format_json=True, - temperature=0.1, - ) - if resp and "response" in resp: - import json - - clean = extract_json(resp["response"]) - if clean: - data = json.loads(clean) - is_safe = data.get("safe", True) - if not is_safe: - logger.warning(f"🚨 [TRAP DETECTED] VLM says: {data.get('reason')}") - return is_safe - except Exception as e: - logger.warning(f"⚠️ [VLM TRAP GUARD] Failed: {e}") - return True - - def classify_screen_content(self, xml_hierarchy: str, target_class: str) -> Optional[str]: - """ - Classifies the current screen content based on learned semantics. - - Zero-Latency Lookup: - 1. Extract semantic 'Ad Marker' signatures. - 2. Query Qdrant vector memory for identical/similar markers. - 3. If no hit, return None (allowing fallback to VLM or legacy). - """ - if not xml_hierarchy: - return None - - # 1. Structural Fingerprinting (extract potential markers) - # We look for nodes that traditionally contain 'sponsored' indicators - # but we don't check for specific strings yet—we let the embedding decide. - candidates = [] - try: - clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_hierarchy).strip() - root = ET.fromstring(clean_xml) - for node in root.iter("node"): - attrib = node.attrib - text = attrib.get("text", "") - desc = attrib.get("content-desc", "") - res_id = attrib.get("resource-id", "") - - # Markers usually sit in small, specific nodes near the header or CTA - if text or desc: - candidates.append(f"{text} {desc} {res_id}") - except Exception: - return None - - if not candidates: - return None - - # 2. Vector Memory Lookup - # We use ContentMemoryDB to see if any of these candidates were previously labeled. - from GramAddict.core.qdrant_memory import ContentMemoryDB - - memory = ContentMemoryDB() - - # Check top candidates (usually markers are short) - for cand in sorted(candidates, key=len)[:10]: - match = memory.get_cached_evaluation(cand, similarity_threshold=0.98) - if match: - logger.info( - f"🧠 [Telepathic] Learned ad marker detected in memory: '{cand}' -> {match['classification']}" - ) - return match["classification"] - - return None - - def _find_best_node_inner( - self, xml_hierarchy: str, intent_description: str, min_confidence: float = 0.82, device=None, **kwargs - ) -> Optional[dict]: - """ - Scans the screen and returns the center coordinates (x, y) of the node - whose embedding is most mathematically similar to the intent. - - Resolution cascade (ordered by speed & reliability): - 1. Positive Memory Cache (past CONFIRMED clicks) - 2. Keyword Fast Path (deterministic string matching) - 3. Vector Similarity Engine (embedding cosine similarity) - 4. Vision Cortex Fallback (VLM, with structural guards) - - All results are PROVISIONAL until the caller confirms via confirm_click(). - Failed clicks should be reported via reject_click(). - """ - logger.debug(f"[_find_best_node_inner] Seeking intent: '{intent_description}'") - - # ── Global Intent Guards ── - intent_lower = intent_description.lower() - if "comment" in intent_lower: - xml_lower = str(xml_hierarchy).lower() - if ( - "comments are turned off" in xml_lower - or "comments on this post" in xml_lower - or "kommentare sind deaktiviert" in xml_lower - or "eingeschränkt" in xml_lower - ): - logger.info("⏭️ [Telepathic] Comments are disabled on this post. Skipping to prevent VLM hallucination.") - return {"x": None, "y": None, "score": 1.0, "semantic": "comments_disabled", "skip": True} - - interactive_nodes = self._extract_semantic_nodes(xml_hierarchy) - if not interactive_nodes: - logger.debug("[_find_best_node_inner] Screen contains no interactable semantic nodes.") - return None - - # Guard against clicking 'Following' when we want to 'Follow' - if "follow" in intent_lower and "follower" not in intent_lower: - for n in interactive_nodes: - res_id = n.get("resource_id", "") - text_lower = (n.get("text", "") or n.get("content_desc", "")).lower().strip() - - # Check absolute ID - if "profile_header_follow_button" in res_id: - if any( - state in text_lower for state in ["following", "gefolgt", "angefragt", "requested", "abonniert"] - ): - logger.info( - f"✅ [Intent Guard] Goal '{intent_description}' fulfilled (Button says '{text_lower}'). Bailing click." - ) - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - # Check generic exact matches on standard toggle strings - elif text_lower in ["following", "gefolgt", "angefragt", "requested", "abonniert"]: - logger.info( - f"✅ [Intent Guard] Goal '{intent_description}' fulfilled (Generic button '{text_lower}'). Bailing click." - ) - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - - # Detect screen height for zone calculations - screen_height = 2400 - if device and hasattr(device, "get_info"): - try: - screen_height = device.get_info().get("displayHeight", 2400) - except Exception: - pass - - if screen_height == 2400 and interactive_nodes: - max_bounds_y = max((n.get("y", 0) + n.get("height", 0) // 2) for n in interactive_nodes) - if max_bounds_y > 2000: - screen_height = max_bounds_y - - # ── Modal Guard ── - # If a bottom sheet or dialog is active, it likely obscures the main navigation tabs. - # We rely strictly on the SAE (Situational Awareness Engine) for 100% autonomous detection - # (via Qdrant cache or LLM reasoning) instead of hardcoded resource IDs. - is_nav_intent = any(k in intent_lower for k in NAV_ZONE_BYPASS_KEYWORDS) - if is_nav_intent and device is not None: - from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType - - sae = SituationalAwarenessEngine(device) - situation = sae.perceive(xml_hierarchy) - if situation == SituationType.OBSTACLE_MODAL: - logger.warning( - f"🛡️ [Modal Guard] SAE detected a modal blocking the screen. Refusing to seek nav-intent '{intent_description}'." - ) - return {"blocked_by_modal": True} - - # ── DM Thread (Forbidden Zone) Guard ── - # If the bot is trapped inside a DM thread, it should NOT attempt to find - # profile-related targets (like the grid or follow button). This prevents the - # "Message-Hijacking" loop discovered in session 2026-04-17. - is_profile_seeking = any( - k in intent_lower for k in ["profile grid", "follow button", "story ring", "grid item", "grid first post"] - ) - is_dm_thread = any( - k in xml_hierarchy for k in ["direct_thread_header", "message_list", "row_thread_composer_edittext"] - ) - - if is_dm_thread and is_profile_seeking: - logger.warning( - f"🛡️ [DM Forbidden Zone] Refusing profile-intent '{intent_description}' inside a DM thread. Forcing navigation fallback." - ) - return {"blocked_by_dm_thread": True} - - # ── Others Profile (Navigation Mislead) Guard ── - # In external profiles, the "Grid/Reels" tabs often mislead VLM into thinking - # they are the main navigation tabs if the bottom bar is suppressed. - is_others_profile = "profile_header_container" in xml_hierarchy or "row_profile_header" in xml_hierarchy - is_home_nav = any(k in intent_lower for k in ["tap home tab", "home feed index"]) - - if is_others_profile and is_home_nav: - # Check if the bottom bar components are actually present. - # Usually: self_profile_tab, home_tab, explore_tab - has_nav_bar = "tab_bar" in xml_hierarchy or "home_tab" in xml_hierarchy or "reels_tab" in xml_hierarchy - if not has_nav_bar: - logger.warning( - f"🛡️ [Profile Nav Guard] Detected 'OthersProfile' with MISSING bottom bar. Blocking hallucination-prone intent '{intent_description}'." - ) - return {"blocked_by_others_profile_trap": True} - - # Pre-filter: Remove structurally implausible nodes and blacklisted mappings - viable_nodes = [] - for node in interactive_nodes: - if not self._structural_sanity_check(node, intent_description, screen_height): - continue - if self._is_blacklisted(intent_description, node["semantic_string"]): - logger.debug( - f"🚫 [Blacklist] Skipping known-bad mapping: '{intent_description}' → '{node['semantic_string']}'" - ) - continue - viable_nodes.append(node) - - if not viable_nodes: - logger.warning(f"[TelepathicEngine] No viable nodes left after filtering for '{intent_description}'") - return None - - # ── App Context Guard: Abort if NOT in Target App ── - if not self._is_instagram_context(interactive_nodes): - if device: - current_app = device._get_current_app() - if current_app != device.app_id: - logger.warning( - f"⚠️ [Context Guard] Not in target app (Current: {current_app}). Aborting AI lookup for '{intent_description}'." - ) - return None - else: - logger.warning(f"⚠️ [Context Guard] Not in target app! Aborting AI lookup for '{intent_description}'.") - return None - - # ══════════════════════════════════════════════════════════════ - # ── Stage 0: DETERMINISTIC Core Navigation Fast Path ── - # THIS MUST BE FIRST. It uses LIVE resource-IDs from the CURRENT - # XML dump. Memory can be poisoned by past notification overlays - # that shifted coordinates. Resource-IDs are ground truth. - # ══════════════════════════════════════════════════════════════ - core_nav_result = self._core_navigation_fast_path(intent_description, viable_nodes) - if core_nav_result: - return core_nav_result - - # ── Stage 0.5: Structural Grid Fast Path ── - # Direct resource-ID + spatial sorting for grid navigation intents. - # This bypasses keyword/vector/VLM entirely — O(n) deterministic. - low_intent = intent_description.lower() - is_grid_intent = any(k in low_intent for k in ["explore grid", "grid item", "first image", "profile grid"]) - if is_grid_intent: - skip = kwargs.get("skip_positions", set()) - grid_result = self._grid_fast_path(intent_description, viable_nodes, skip_positions=skip) - if grid_result: - return grid_result - - # ── Stage 1: Positive Memory Cache (CONFIRMED past clicks) ── - self._memory = self._load_json(MEMORY_FILE) # Reload for freshness - if intent_description in self._memory: - known_semantics = self._memory[intent_description] - - # Anti-poisoning: for dynamic intents, strip text/desc to match structural signatures - dynamic_intents = { - "tap post username", - "tap post author", - "tap story ring avatar", - "tap feed item", - "explore grid", - "grid item", - "first image", - "profile grid", - "tap user profile", - "post media content", - } - is_dynamic = any(d in intent_description.lower() for d in dynamic_intents) - - for n in viable_nodes: - sem_str = n["semantic_string"] - if is_dynamic: - sem_str = re.sub(r"(text|description):\s*'[^']*',?\s*", "", sem_str).strip() - sem_str = re.sub(r",\s*id context:", "id context:", sem_str).strip() - - # Check if this semantic string is in memory (handles both old list format and new dict format) - if isinstance(known_semantics, dict) and sem_str in known_semantics: - confidence = known_semantics[sem_str] - if confidence < 0.5: - continue # Stale, force LLM/Vector fallback - - # Prevent un-liking - if "like" in intent_description.lower() and re.search( - r"\b(liked|gefällt mir nicht mehr)\b", sem_str.lower() - ): - logger.info("⏭️ [Memory] Post is already Liked. Skipping tap to prevent un-liking.") - return {"x": None, "y": None, "score": 1.0, "semantic": "already_liked", "skip": True} - - # Prevent un-following - if "follow" in intent_description.lower() and re.search( - r"\b(following|requested|folgst du|angefragt|gefolgt)\b", sem_str.lower() - ): - logger.info( - "⏭️ [Memory] User is already Followed/Requested. Skipping tap to prevent opening the Favorites menu." - ) - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - - logger.debug( - f"🧠 [Confirmed Memory] Instant recall: '{intent_description}' → {sem_str} (Conf: {confidence:.2f})" - ) - self._track_click(intent_description, n) - return { - "x": n["x"], - "y": n["y"], - "score": confidence, - "semantic": f"Memory Match: {sem_str}", - "source": "memory", - "original_attribs": n.get("original_attribs", {}), - } - elif isinstance(known_semantics, list) and sem_str in known_semantics: - # Legacy fallback logic for old JSON list format - if "like" in intent_description.lower() and re.search( - r"\b(liked|gefällt mir nicht mehr)\b", sem_str.lower() - ): - return {"x": None, "y": None, "score": 1.0, "semantic": "already_liked", "skip": True} - if "follow" in intent_description.lower() and re.search( - r"\b(following|requested|folgst du|angefragt|gefolgt)\b", sem_str.lower() - ): - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - - self._track_click(intent_description, n) - return { - "x": n["x"], - "y": n["y"], - "score": 1.0, - "semantic": f"Memory Match: {sem_str}", - "source": "memory", - "original_attribs": n.get("original_attribs", {}), - } - - # ── Stage 1.5: Deterministic Keyword Fast Path ── - fast_path_result = self._keyword_match_score(intent_description, viable_nodes) - if fast_path_result: - return fast_path_result - - # ── Stage 2: Vector Similarity Engine ── - intent_vec = self._get_cached_embedding(intent_description, is_intent=True) - if intent_vec: - scored_nodes = [] - for node in viable_nodes: - node_vec = self._get_cached_embedding(node["semantic_string"]) - if not node_vec: - continue - score = self._cosine_similarity(intent_vec, node_vec) - scored_nodes.append((node, score)) - - # Sort by score descending - scored_nodes.sort(key=lambda x: x[1], reverse=True) - - # Update viable_nodes so that the VLM fallback gets the top semantic candidates - viable_nodes = [n for n, s in scored_nodes] - - # Among high-confidence matches, prefer smaller/more atomic elements - if scored_nodes and scored_nodes[0][1] >= min_confidence: - # Get all nodes within 0.05 of the top score - top_score = scored_nodes[0][1] - top_tier = [(n, s) for n, s in scored_nodes if s >= top_score - 0.05] - - # Among equally-scored candidates, prefer the smallest (most atomic) - top_tier.sort(key=lambda x: x[0].get("area", 999999)) - best_node, best_score = top_tier[0] - - # Prevent un-liking - if "like" in intent_description.lower() and re.search( - r"\b(liked|gefällt mir nicht mehr)\b", best_node["semantic_string"].lower() - ): - logger.info("⏭️ [Telepathic] Post is already Liked. Skipping.") - return {"x": None, "y": None, "score": 1.0, "semantic": "already_liked", "skip": True} - - # Prevent un-following - if "follow" in intent_description.lower() and re.search( - r"\b(following|requested|folgst du|angefragt|gefolgt)\b", best_node["semantic_string"].lower() - ): - logger.info( - "⏭️ [Telepathic] User is already Followed/Requested. Skipping to prevent opening the Favorites menu." - ) - return {"x": None, "y": None, "score": 1.0, "semantic": "already_followed", "skip": True} - - logger.info( - f"✨ [Telepathic Match] '{intent_description}' ➔ {best_node['semantic_string']} (Score: {best_score:.3f})" - ) - self._track_click(intent_description, best_node) - return { - "x": best_node["x"], - "y": best_node["y"], - "score": best_score, - "semantic": best_node["semantic_string"], - "source": "vector", - "original_attribs": best_node.get("original_attribs", {}), - } - elif scored_nodes: - logger.warning( - f"⚠️ [Telepathic] Low confidence ({scored_nodes[0][1]:.3f} < {min_confidence}) for '{intent_description}'." - ) - - # ── Stage 3: Telepathic LLM Fallback (Text-Based XML Reasoning) ── - if device: - logger.info(f"🧠 [Agentic Fallback] Activating structural LLM reasoning for: '{intent_description}'") - goal = kwargs.get("goal", None) - return self._vision_cortex_fallback(intent_description, viable_nodes, device, screen_height, goal=goal) - - return None - - # ────────────────────────────────────────────── - # Click Tracking & Feedback Loop - # ────────────────────────────────────────────── - - def evaluate_grid_visuals(self, device, persona_interests: list[str]) -> Optional[dict]: - """ - [Phase 2] High-fidelity grid evaluation. - Instead of clicking the first available post, uses vision to pick the best match - against configured persona interests. - """ - logger.info(f"👁️ [Vision Core] Analyzing grid aesthetics against niche interests: {persona_interests}...") - - xml = device.dump_hierarchy() - nodes = self._extract_semantic_nodes(xml) - - # Identify grid nodes (posts) dynamically without hardcoded IDs - # Look for interactive elements that are primarily visual (no text) or have photo/video descriptions - grid_nodes = [] - for n in nodes: - orig = n.get("original_attribs", {}) - desc = orig.get("content-desc", "").lower() - has_text = bool(orig.get("text", "")) - - # A grid post is typically a visual block without raw UI text, - # or explicitly labeled as photo/video by accessibility layers. - if not has_text and ("photo" in desc or "video" in desc or "image" in desc or "post" in desc or desc == ""): - grid_nodes.append(n) - - if not grid_nodes: - logger.warning("👁️ [Vision Core] No grid items found to evaluate. Falling back to default navigation.") - return None - - # Sort them Top-to-Bottom, Left-to-Right to match indexing [0-8] - grid_nodes.sort(key=lambda n: (round(n["y"] / 20) * 20, n["x"])) - - # Limit to the top 9 items (3x3) to keep context manageable for VLM - grid_nodes = grid_nodes[:9] - # Take a screenshot - try: - screenshot_b64 = device.get_screenshot_b64() - except Exception as e: - logger.error(f"👁️ [Vision Core] Failed to capture screenshot: {e}") - return None - - # Format the nodes for the vision model context - simplified_nodes = [] - import re - - for i, node in enumerate(grid_nodes): - # Parse bounds string like "[0,123][144,300]" - b_str = node.get("bounds", "[0,0][0,0]") - coords = [int(x) for x in re.findall(r"\d+", b_str)] - if len(coords) == 4: - simplified_nodes.append({"index": i, "bounds": [coords[0], coords[1], coords[2], coords[3]]}) - - system_prompt = ( - "You are an aesthetic evaluation agent for Instagram with a professional eye for niche alignment. " - "You are looking at a 3x3 grid of post thumbnails. " - "Your goal is to pick the image index [0-8] that BEST matches the provided niche interests." - ) - - user_prompt = ( - f"Niche Interests: {', '.join(persona_interests)}\n\n" - f"Grid Elements (indices and bounding boxes):\n{json.dumps(simplified_nodes)}\n\n" - "Return a JSON object: {\"best_index\": number, \"reason\": \"brief explanation of why this image fits the niche\"}\n" - "If none fit, pick the most generic/high-quality one." - ) - - try: - from GramAddict.core.llm_provider import query_llm - - args = device.args - model = getattr(args, "ai_telepathic_model", "llama3.2-vision") - url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - - # Use a slightly higher temperature for aesthetic judging - resp_dict = query_llm( - url=url, - model=model, - prompt=user_prompt, - system=system_prompt, - images_b64=[screenshot_b64], - format_json=True, - temperature=0.4, - ) - - if resp_dict and "response" in resp_dict: - from GramAddict.core.llm_provider import extract_json - - clean_json = extract_json(resp_dict["response"]) - if clean_json: - data = json.loads(clean_json) - if isinstance(data, list) and len(data) > 0: - data = data[0] - if isinstance(data, dict): - idx = data.get("best_index") - if idx is not None and 0 <= idx < len(grid_nodes): - chosen = grid_nodes[idx] - logger.info( - f"✅ [Vision Match] Cell {idx} chosen: {data.get('reason')}", - extra={"color": Fore.GREEN}, - ) - self._track_click(f"Visual Grid Selection ({idx})", chosen) - return { - "x": chosen["x"], - "y": chosen["y"], - "score": 0.99, - "semantic": f"Visual match {idx}: {data.get('reason')}", - "source": "vlm_grid", - "original_attribs": chosen.get("original_attribs", {}), - } - except Exception as e: - logger.error(f"👁️ [Vision Core] Grid evaluation failed: {e}") - - return None - - def evaluate_post_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: - """ - Takes a screenshot of the current post/reel and asks the VLM to evaluate its - aesthetic quality and niche alignment. Returns a dict with quality_score and matches_niche. - """ - logger.info("👁️ [Vision Core] Capturing post screenshot for Content Vibe Check...", extra={"color": "\033[36m"}) - - try: - screenshot_b64 = device.get_screenshot_b64() - except Exception as e: - logger.error(f"👁️ [Vision Core] Failed to capture post screenshot: {e}") - return None - - system_prompt = ( - "You are a strict aesthetic evaluator for an Instagram growth agent. " - "You are looking at a screenshot of a single Instagram post or Reel. " - "Evaluate the visual content, subject matter, and aesthetic quality. " - 'Return a JSON object: {"quality_score": number (1-10), "matches_niche": boolean, "reason": "string"}. ' - "Extremely generic, spammy, or unrelated content should get a low score (< 5). " - "High quality, aesthetic, or highly niche-aligned content gets >= 7." - ) - - user_prompt = ( - f"Niche/Interests: {', '.join(persona_interests) if persona_interests else 'Aesthetic / General Quality'}\n\n" - "Evaluate the provided post screenshot strictly." - ) - - try: - from GramAddict.core.llm_provider import query_llm - - args = getattr(device, "args", None) - model = getattr(args, "ai_telepathic_model", "llama3.2-vision") if args else "llama3.2-vision" - url = ( - getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - if args - else "http://localhost:11434/api/generate" - ) - - resp_dict = query_llm( - url=url, - model=model, - prompt=user_prompt, - system=system_prompt, - format_json=True, - images_b64=[screenshot_b64], - max_tokens=200, - temperature=0.2, - timeout=45, - ) - - if resp_dict and "response" in resp_dict: - import json - - try: - res_text = resp_dict["response"] - if res_text.startswith("```json"): - res_text = res_text[7:] - if res_text.endswith("```"): - res_text = res_text[:-3] - - data = json.loads(res_text.strip()) - return data - except Exception as e: - logger.error( - f"👁️ [Vision Core] Failed to parse JSON from VLM: {e}\nResponse: {resp_dict['response']}" - ) - except Exception as e: - logger.error(f"👁️ [Vision Core] Error calling VLM for post check: {e}") - - return None - - def evaluate_profile_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: - """ - [Phase 1] High-fidelity Target Profile Vibe Check. - Takes a screenshot of the user's profile and asks the VLM to score their aesthetic quality - and niche alignment to preemptively filter out generic/spammy users. - """ - logger.info("👁️ [Vision Core] Capturing profile screenshot for Vibe Check...", extra={"color": "\\033[36m"}) - - try: - screenshot_b64 = device.get_screenshot_b64() - except Exception as e: - logger.error(f"👁️ [Vision Core] Failed to capture profile screenshot: {e}") - return None - - system_prompt = ( - "You are a strict aesthetic evaluator for an Instagram growth agent. " - "You are looking at a screenshot of an Instagram user's profile. " - "Evaluate their bio, profile picture, and the visible grid posts. " - 'Return a JSON object: {"quality_score": number (1-10), "matches_niche": boolean, "reason": "string"}. ' - "Extremely generic, spammy, or empty profiles should get a low score (< 5). " - "High quality, aesthetic, or highly personalized profiles get >= 7." - ) - - user_prompt = ( - f"Niche/Interests: {', '.join(persona_interests) if persona_interests else 'Aesthetic / General Quality'}\n\n" - "Evaluate the provided profile screenshot strictly." - ) - - try: - from GramAddict.core.llm_provider import query_llm - - args = getattr(device, "args", None) - model = getattr(args, "ai_telepathic_model", "llama3.2-vision") if args else "llama3.2-vision" - url = ( - getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - if args - else "http://localhost:11434/api/generate" - ) - - resp_dict = query_llm( - url=url, - model=model, - prompt=user_prompt, - system=system_prompt, - images_b64=[screenshot_b64], - format_json=True, - temperature=0.4, - ) - - if resp_dict and "response" in resp_dict: - import json - - from GramAddict.core.llm_provider import extract_json - - clean_json = extract_json(resp_dict["response"]) - if clean_json: - data = json.loads(clean_json) - score = data.get("quality_score", 5) - niche = data.get("matches_niche", True) - reason = data.get("reason", "No reason provided") - - logger.info(f"✨ [Vibe Check] Score: {score}/10 | Niche: {niche} | Reason: {reason}") - return data - - return None - - except Exception as e: - logger.error(f"👁️ [Vision Core] Failed to call VLM for profile vibe check: {e}") - return None - - def _grid_fast_path( - self, intent_description: str, viable_nodes: list, skip_positions: set = None - ) -> Optional[dict]: - """ - Deterministic grid navigation: structurally filters for likely grid items (images, no text), - sorts by (y, x), and returns the topmost-leftmost candidate. - - skip_positions: set of (x, y) tuples to skip on retry, ensuring - the Fast-Path doesn't re-click a position that already failed. + Legacy proxy for extracting semantic nodes. + If a query is provided, it tries to find the best node matching it. + If no query is provided, it returns all clickable nodes to support zero-latency feed loops. """ + if semantic_query: + best_node = self.find_best_node(xml_string, semantic_query, **kwargs) + return [best_node] if best_node else [] + + # If no query, return all interactable nodes as dicts (used by zero latency loops) + root = self._parser.parse(xml_string) + if not root: + return [] + nodes = self._parser.get_clickable_nodes(root) + return [self._translate_node(n) for n in nodes] + + def _grid_fast_path(self, intent_description: str, nodes: list, skip_positions: set = None) -> Optional[dict]: if skip_positions is None: skip_positions = set() - # Structure-based grid detection: interactive visual elements with no text - grid_nodes = [] - for n in viable_nodes: - desc = n.get("content_desc", "").lower() - res_id = n.get("resource_id", "").lower() - has_text = bool(n.get("text", "")) - - # 🛡️ Defensive Guard: Block profile pictures, header items, and anything too high up. - if "profile" in res_id or n.get("y", 0) < 400: - continue - - if not has_text and ("photo" in desc or "video" in desc or "image" in desc or "post" in desc or desc == ""): - grid_nodes.append(n) - - if not grid_nodes: - return None - - # Preference logic with rounding to group items clearly in the same row: - # 1. Round Y to nearest 5px to handle minor layout discrepancies - # 2. Non-NAF is better than NAF (accessibility friendly) - # 3. Containers (grid_card_layout_container) preferred over inner buttons - grid_nodes.sort( - key=lambda n: ( - round(n["y"] / 5) * 5, - n["x"], - n.get("naf", False), # False (0) is better than True (1) - -n["area"], # Larger area (containers) preferred among equal y/x - ) - ) - - # Filter out previously-failed positions - for candidate in grid_nodes: - pos = (candidate["x"], candidate["y"]) - if pos in skip_positions: - continue - - logger.info( - f"⚡ [Grid Fast-Path] Matched '{intent_description}' → {candidate['semantic_string']} (y={candidate['y']})" - ) - self._track_click(intent_description, candidate) - return { - "x": candidate["x"], - "y": candidate["y"], - "score": 0.98, - "semantic": candidate["semantic_string"], - "source": "grid_fastpath", - "original_attribs": candidate.get("original_attribs", {}), - } - - # All grid nodes were in skip_positions - logger.warning( - f"⚠️ [Grid Fast-Path] All grid positions exhausted for '{intent_description}'. Falling through to VLM." - ) + if "first image in explore grid" in intent_description.lower(): + grid_items = [ + n + for n in nodes + if n.get("y", 9999) < 2000 + and ( + "grid card layout container" in (n.get("semantic_string", "") or "").lower() + or "image button" in (n.get("semantic_string", "") or "").lower() + ) + and (n.get("x", -1), n.get("y", -1)) not in skip_positions + ] + if grid_items: + # Sort by y (row) then by x (col) + grid_items.sort(key=lambda n: (n.get("y", 9999), n.get("x", 9999))) + return grid_items[0] return None - def _core_navigation_fast_path(self, intent_description: str, viable_nodes: list) -> Optional[dict]: - """ - [Phase 2] Hardened Fast Path with Qdrant Self-Learning. - Absolutely deterministic resource-ID targeting for core application navigation - (like direct messages or post usernames). We query Qdrant first. If empty, - we MUST fall back to Telepathic semantic discovery. - This enforces a 100% autonomous Blank Start architecture. - """ - intent_description.lower().strip() - - # 0. Query Qdrant Memory first! - mem = self.ui_memory.retrieve_memory(intent_description, "", similarity_threshold=0.9, exact_only=True) - if mem and isinstance(mem, dict): - if mem.get("action") == "tap" and mem.get("resource_id"): - learned_res_id = mem.get("resource_id").lower() - learned_res_id_underscores = learned_res_id.replace(" ", "_") - for n in viable_nodes: - raw_id = n.get("resource_id", "").lower() - if learned_res_id in raw_id or learned_res_id_underscores in raw_id: - logger.info( - f"🧠 [TelepathicEngine] Fast-Path resolved '{intent_description}' via Qdrant -> '{learned_res_id}'", - extra={"color": "\\033[36m"}, - ) - self._track_click(intent_description, n) - return { - "x": n["x"], - "y": n["y"], - "score": 1.0, - "semantic": n["semantic_string"], - "source": "qdrant_nav", - "original_attribs": n.get("original_attribs", {}), - } - return None - - def _track_click(self, intent: str, node: dict): - """Records what we're about to click so confirm/reject can reference it.""" - TelepathicEngine._last_click_context = { - "intent": intent, - "semantic_string": node["semantic_string"], - "x": node["x"], - "y": node["y"], - "timestamp": time.time(), - } + # ────────────────────────────────────────────── + # Action Memory Delegation + # ────────────────────────────────────────────── def confirm_click(self, intent: str = None): - """ - Called by the interaction layer AFTER verifying the click produced the expected result. - Stores the mapping as a confirmed positive learning. - - Usage: - result = telepathic.find_best_node(xml, "tap like button", device=device) - _humanized_click(device, result["x"], result["y"]) - # ... verify the like actually happened ... - telepathic.confirm_click("tap like button") - """ - ctx = TelepathicEngine._last_click_context - if not ctx: - return - - actual_intent = intent or ctx["intent"] - sem = ctx["semantic_string"] - - # ── Qdrant Structural Persistence ── - # Extract resource-id from semantic string to learn structural navigation - if hasattr(self, "ui_memory") and self.ui_memory and self.ui_memory.is_connected: - res_id_match = re.search(r"id context:\s*'([^']+)'", sem) - if res_id_match: - learned_res_id = res_id_match.group(1) - self.ui_memory.store_memory( - intent=actual_intent, - xml_context="", # UI Memory uses intent hashes - solution={"action": "tap", "resource_id": learned_res_id}, - ) - - # ── Anti-Poisoning Guard for JSON Memory ── - dynamic_intents = { - "tap post username", - "tap post author", - "tap story ring avatar", - "tap feed item", - "explore grid", - "grid item", - "first image", - "profile grid", - "tap user profile", - "post media content", - } - is_dynamic = any(d in actual_intent.lower() for d in dynamic_intents) - if is_dynamic: - # Strip dynamic text and description to prevent overfitting - sem = re.sub(r"(text|description):\s*'[^']*',?\s*", "", sem).strip() - sem = re.sub(r",\s*id context:", "id context:", sem).strip() - - if not sem: - logger.debug( - f"⚠️ [Confirmed Learning] Semantic string became empty after stripping dynamic content for '{actual_intent}'. Skipping JSON cache." - ) - TelepathicEngine._last_click_context = None - return - - # Add to positive memory with 100% confidence - if actual_intent not in self._memory: - self._memory[actual_intent] = {} - elif isinstance(self._memory[actual_intent], list): - # Migrate legacy list to new dict format - legacy_list = self._memory[actual_intent] - self._memory[actual_intent] = {k: 1.0 for k in legacy_list} - - # Boost/Set confidence to 1.0 - self._memory[actual_intent][sem] = 1.0 - self._save_json(MEMORY_FILE, self._memory) - logger.debug(f"✅ [Confirmed Learning] Stored/Boosted: '{actual_intent}' → '{sem}' (Confidence: 1.0)") - - # Remove from blacklist if it was there (rehabilitation) - if actual_intent in self._blacklist and sem in self._blacklist[actual_intent]: - self._blacklist[actual_intent].remove(sem) - self._save_json(BLACKLIST_FILE, self._blacklist) - logger.debug(f"🔄 [Rehabilitation] Removed from blacklist: '{actual_intent}' → '{sem}'") - - # CLEAR context after confirmation to prevent double learning - TelepathicEngine._last_click_context = None + self._memory.confirm_click(intent) def reject_click(self, intent: str = None): - """ - Called by the interaction layer when the click did NOT produce the expected result. - Adds the mapping to the blacklist (negative learning) so it's never tried again. - Also removes it from positive memory if it was cached there. - """ - ctx = TelepathicEngine._last_click_context - if not ctx: - return - - actual_intent = intent or ctx["intent"] - sem = ctx["semantic_string"] - - # ── Anti-Poisoning Guard ── - # Structural UI elements (Home Tab, Like Button, etc.) should NEVER be globally blacklisted - # because they are essential for navigation and are unlikely to cause app-drift (only Ads do). - structural_intents = { - "tap home tab", - "tap reels tab", - "tap profile tab", - "tap direct message icon inbox", - "tap explore tab", - "tap newsfeed_tab", - "tap like button", - "tap comment button", - "tap share button", - "tap story ring avatar", - } - - has_text = bool(re.search(r"(? bool: - """ - Hardened verification. Does NOT rely on raw XML changes. - Inspects the post-click XML for semantic markers that prove the intent worked. - """ - _ctx = TelepathicEngine._last_click_context - # If no context is set, we can't do positional consistency, - # but we MUST still enforce intent-specific markers below. - - # 1. Positional Consistency (Atomic Guard) - # If the screen changed so much that NO elements are near the original click, - # it might be a context-switch (navigation success). - - # 2. Intent-Specific Markers - low_intent = intent.lower() - low_xml = post_click_xml.lower() - - # 0. Global Hallucination Guards - # If we didn't intend to share/send and the share sheet opened, the click was a hallucination. - if "share" not in low_intent and "send" not in low_intent and "share_sheet" in low_xml: - logger.warning( - f"❌ [Semantic Verification] FAILED: Hallucinated click opened the direct share sheet. ({intent})" - ) - return False - - if "poll" not in low_intent and "survey" not in low_intent and ("_poll_" in low_xml or "survey_" in low_xml): - logger.warning(f"❌ [Semantic Verification] FAILED: Hallucinated click opened a survey/poll. ({intent})") - return False - - # Success markers for common actions - if "like" in low_intent: - # Check for "Liked" or "gefällt mir nicht mehr" in content-desc or text - marker_found = re.search(r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", low_xml) - if marker_found: - logger.debug("✅ [Semantic Verification] Success confirmed: 'Liked' state detected.") - return True - else: - logger.warning("❌ [Semantic Verification] FAILED: Post does not report 'Liked' state after click.") - return False - - if "follow" in low_intent: - # ── Anti-Poisoning Guard ── - # Tapping a follow button should NOT transition the app from the feed to a profile screen. - # If it did, we hallucinated and clicked a profile name instead of a follow button. - if "row_feed_photo_profile_name" in low_xml or "profile_header" in low_xml: - # If we were previously NOT on a profile, but now we are, it's a hallucination. - # Since we don't have perfect state here, we rely on the fact that tapping "Follow" - # on a profile keeps you on the same profile layout. If the layout radically changes, - # it's safer to mark as inconclusive than falsely succeed. - # For now, we strictly check for the following marker. - pass - - # Check if button changed to "Following" or "Requested" or "Abonniert" (German) - marker_found = re.search(r"\b(following|requested|folgst du|angefragt|abonniert)\b", low_xml) - if marker_found: - # Extra Guard: Ensure we didn't just open someone's profile from the feed by mistake - # which also has "Following" on it. We check if the screen is still the same rough layout. - if _ctx and "row_feed" in _ctx["semantic_string"].lower() and "profile_header" in low_xml: - logger.warning( - "❌ [Semantic Verification] FAILED: Tapping 'Follow' navigated to a profile page! (Hallucination)" - ) - return False - - logger.debug("✅ [Semantic Verification] Success confirmed: 'Following/Requested' state detected.") - return True - else: - logger.warning("❌ [Semantic Verification] FAILED: Profile does not report 'Following' state.") - return False - - if any(k in low_intent for k in ["explore grid", "profile grid", "first image", "grid item"]): - # Clicking a grid item MUST open a post view. - # Posts have feed markers. Reels/clips have their own markers. - post_markers = [ - # Normal feed posts - "row_feed_button_like", - "row_feed_button_comment", - "row_feed_button_share", - "row_feed_comment_textview_layout", - "row_feed_view_group", - "row_feed_photo_profile_name", - "row_feed_photo_imageview", - # Reels / Clips - "clips_media_component", - "clips_viewer", - "clips_like_button", - "clips_comment_button", - "reel_viewer", - "clips_music_attribution", - # Carousel / Gallery - "carousel_page_indicator", - "media_set_page_indicator", - # Generic post markers - "action_bar_original_title", - "media_header_user", - ] - marker_found = any(m in low_xml for m in post_markers) - if marker_found: - logger.debug("✅ [Semantic Verification] Success confirmed: Post/Reel opened from grid.") - return True - else: - grid_markers = [ - "explore_tab", - "explore_grid", - "grid_card_layout_container", - "image_button", - "profile_tabs_container", - ] - if any(m in low_xml for m in grid_markers): - logger.warning( - "⚠️ [Semantic Verification] INCONCLUSIVE: Bot is STILL on the grid. Tap may have failed to register." - ) - return None - - logger.warning("❌ [Semantic Verification] FAILED: Grid tap did not open a valid post view.") - return False - - # For general navigation, raw XML change is still the fallback - # (covered by the caller in q_nav_graph for now) - return True + pre_click_xml = "" + if self._memory._last_click_context: + pre_click_xml = self._memory._last_click_context.get("xml_context", "") + return self._memory.verify_success(intent, pre_click_xml, post_click_xml) # ────────────────────────────────────────────── - # Vision Cortex Fallback (VLM) + # Semantic Evaluator Delegation # ────────────────────────────────────────────── - def _vision_cortex_fallback( - self, intent: str, nodes: list[dict], device, screen_height: int = 2400, goal: str = None - ) -> Optional[dict]: - """ - Uses a Language Model to identify the correct node from parsed screen XML - when embeddings are insufficient. Opt-in Native Vision Processing via Device Screenshots! + def evaluate_grid_visuals(self, device, persona_interests: list[str]) -> Optional[dict]: + root = self._parser.parse(device.dump_hierarchy()) + if not root: + return None - Guards are STRUCTURAL (size, position, class) not ID-based. - Learning happens via the confirm/reject feedback loop, not here. - """ - try: - from GramAddict.core.config import Config - - args = getattr(Config(), "args", None) - use_vision = getattr(args, "ai_vision_navigation", False) if args else False - images_payload = None - - # Ensure screen_height is a safe integer to avoid MagicMock TypeError in tests - import numbers - - if not isinstance(screen_height, numbers.Number): - screen_height = 2400 - else: - try: - screen_height = int(screen_height) - except (ValueError, TypeError): - screen_height = 2400 - - if use_vision and device is not None: - try: - logger.debug("👁️ [Vision Inference] Capturing screen for spatial understanding...") - img_obj = device.screenshot() - if img_obj: - import io - - if hasattr(img_obj, "save"): - buf = io.BytesIO() - img_obj.save(buf, format="JPEG") - raw_bytes = buf.getvalue() - elif isinstance(img_obj, bytes): - raw_bytes = img_obj - else: - try: - raw_bytes = bytes(img_obj) - except Exception: - logger.warning( - f"👁️ [Vision Inference] Could not convert screenshot object {type(img_obj)} to bytes." - ) - raw_bytes = None - - if raw_bytes: - b64_str = base64.b64encode(raw_bytes).decode("utf-8") - images_payload = [b64_str] - except Exception as e: - logger.warning(f"⚠️ [Vision Inference] Failed to capture or encode screenshot: {e}") - - # ── Instant Intent Fulfillment (Pre-VLM) ── - # If we are looking for 'follow' and we see 'Following', 'Requested', or 'Follow back', - # we consider the intent ALREADY FULFILLED and skip to avoid opening menus. - intent_lower = intent.lower() - if any(k in intent_lower for k in ["follow"]): - for i, node in enumerate(nodes[:50]): - text_lower = (node.get("text", "") or node.get("content_desc", "")).lower() - if any( - x in text_lower - for x in ["following", "gefolgt", "angefragt", "requested", "follow back", "zurückfolgen"] - ): - logger.info( - f"✅ [Intent Guard] Goal '{intent}' already fulfilled by node {i} ('{text_lower}'). Bailing click." - ) - return {"index": i, "skip": True, "reason": "Already fulfilled"} - - # ── VLM Context Optimization (Top 30 scoring elements only to prevent Timeouts) ── - candidates = [(i, n) for i, n in enumerate(nodes)] - candidates.sort(key=lambda x: x[1].get("score", 0.0), reverse=True) - - simplified_nodes = [] - for orig_idx, n in candidates[:30]: - simplified_nodes.append( - {"index": orig_idx, "bounds": n["raw_bounds"], "semantic": n["semantic_string"]} - ) - - # Get model config - args = getattr(device, "args", None) - model = getattr(args, "ai_telepathic_model", "llama3.2:1b") if args else "llama3.2:1b" - url = ( - getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate") - if args - else "http://localhost:11434/api/generate" - ) - - # --- Model Trust Logging --- - from GramAddict.core.benchmark_guard import BENCHMARKS_FILE - - trust_log = f"Using {model}" - try: - if os.path.exists(BENCHMARKS_FILE): - with open(BENCHMARKS_FILE, "r") as f: - bench_data = json.load(f).get("models", {}).get(model, {}) - if bench_data: - score = bench_data.get("telepathic_score", 0) - passed = "PASS" if bench_data.get("passed_all", False) else "FAIL" - unsuitable = bench_data.get("is_unsuitable", False) - trust_level = ( - "HIGH" - if score >= 80 and not unsuitable - else "MEDIUM" - if score >= 50 and not unsuitable - else "LOW/UNSAFE" - ) - trust_log += f" [Benchmark: {score}/100 | {passed} | Trust: {trust_level}]" - if unsuitable: - logger.error(f"⛔ [Safety Alert] {model} is marked as UNSUITABLE for this task!") - except Exception: - pass - logger.info(f"🧠 [Telepathic] Intent: '{intent}' -> {trust_log}") - # --------------------------- - - system_prompt = ( - "You are an Android UI expert. Identify the correct element index to tap based on the provided JSON.\n" - "Rules:\n" - "1. Output ONLY a raw JSON object.\n" - "2. NO markdown formatting, NO triple backticks, NO explanation.\n" - '3. Format: {"index": number, "reason": "string"}\n' - '4. If no element matches, return {"index": -1, "reason": "no match"}\n' - "5. FATAL RULES: NEVER select 'Share', 'Send post', 'Poll', or 'Survey' buttons UNLESS the intent explicitly commands it!\n" - "6. ACCOUNT SAFETY: NEVER select buttons that modify account state (Favorite, Mute, Block, Unfollow, Restrict) unless specifically commanded." - ) - - if goal: - user_prompt = ( - f"Your overarching ultimate GOAL is to: '{goal}'.\n\n" - f"Which element should I tap to progress towards this goal? The specific short-term intent requested was: {intent}\n\n" - f"Elements:\n{json.dumps(simplified_nodes, indent=1)}\n\n" - "Rules:\n" - "- Pick the SMALLEST, most specific button or icon\n" - "- NEVER pick large containers, full-screen views, or recycler views\n" - "- NEVER pick system icons (wifi, battery, status bar, clock, notifications)\n" - "- IGNORE BOTTOM NAVIGATION TABS (Home, Search, Reels, Message, Profile) if the intent is to interact with a post or comment.\n" - "- SPATIAL GUARD: Elements in the TOP 10% (Y < 240) are USUALLY part of the system status bar. ELEMENT MUST BE BELOW THIS UNLESS IT IS A STORY CIRCLE.\n" - "- SPATIAL GUARD: Elements in the BOTTOM 10% (Y > 2160) are USUALLY part of the navigation bar. ELEMENT MUST BE ABOVE THIS UNLESS IT IS A NAVIGATION TAB.\n" - "- A 'Comment input' is usually an EditText or a region near the bottom but ABOVE the navigation bar.\n" - "- A 'story tray' or 'story ring' is ALWAYS located at the very TOP of the screen (low Y coordinates).\n" - "- NAVIGATION TABS (Home, Explore, Reels, News, Profile) are ALWAYS in the BOTTOM zone (Y coordinates > 0.90 of screen height).\n" - 'Return: {"index": number, "reason": "..."}' - ) - else: - user_prompt = ( - f"Which element should I tap to: {intent}\n\n" - f"Elements:\n{json.dumps(simplified_nodes, indent=1)}\n\n" - "Rules:\n" - "- Pick the SMALLEST, most specific button or icon\n" - "- NEVER pick large containers, full-screen views, or recycler views\n" - "- NEVER pick system icons (wifi, battery, status bar, clock, notifications)\n" - "- IGNORE BOTTOM NAVIGATION TABS (Home, Search, Reels, Message, Profile) if the intent is to interact with a post or comment.\n" - "- SPATIAL GUARD: Elements in the TOP 10% (Y < 240) are USUALLY part of the system status bar. ELEMENT MUST BE BELOW THIS UNLESS IT IS A STORY CIRCLE.\n" - "- SPATIAL GUARD: Elements in the BOTTOM 10% (Y > 2160) are USUALLY part of the navigation bar. ELEMENT MUST BE ABOVE THIS UNLESS IT IS A NAVIGATION TAB.\n" - "- A 'Comment input' is usually an EditText or a region near the bottom but ABOVE the navigation bar.\n" - "- A 'story tray' or 'story ring' is ALWAYS located at the very TOP of the screen (low Y coordinates).\n" - "- NAVIGATION TABS (Home, Explore, Reels, News, Profile) are ALWAYS in the BOTTOM zone (Y coordinates > 0.90 of screen height).\n" - 'Return: {"index": number, "reason": "..."}' - ) - - resp_str = query_telepathic_llm(model, url, system_prompt, user_prompt, images_b64=images_payload) - data = json.loads(resp_str) - - # ── Robustness: Handle list responses from LLM ── - if isinstance(data, list): - if len(data) > 0 and isinstance(data[0], dict): - data = data[0] - else: - logger.error(f"VLM returned unexpected list format: {data}") - return None - - if not isinstance(data, dict): - logger.error(f"VLM returned non-dict response: {type(data)}") - return None - - idx = data.get("index") - if idx is not None and 0 <= idx < len(nodes): - match = nodes[idx] - - # ── Structural Guard 1: Size ── - is_media_intent = any(k in intent.lower() for k in ["video", "photo", "reel", "media", "post"]) - if match.get("area", 0) > MAX_CONTAINER_AREA and not is_media_intent: - logger.warning( - f"🛡️ [Structural Guard] VLM selected oversized element " - f"({match.get('width')}x{match.get('height')}): {match['semantic_string']}. REJECTING." - ) - dump_ui_state( - device, - "vlm_hallucination", - { - "intent": intent, - "rejected_node": match["semantic_string"], - "node_size": f"{match.get('width')}x{match.get('height')}", - "vlm_index": idx, - }, - ) - return None - - # ── Structural Guard 2: Position (status / nav bar / tab zones) ── - if match.get("y", 0) < screen_height * STATUS_BAR_ZONE: - logger.warning( - f"🛡️ [Structural Guard] VLM selected element in status bar zone: {match['semantic_string']}. REJECTING." - ) - return None - - is_nav_intent = any(k in intent.lower() for k in NAV_ZONE_BYPASS_KEYWORDS) - is_tab_intent = any(k in intent.lower() for k in NAV_TAB_KEYWORDS) - - # NAVIGATION TAB ENFORCEMENT: - # Real navigation tabs (Home, Search, Reels, Store, Profile) are ALWAYS in the bottom zone. - # If the bot is looking for a tab, forbid results that are too high up. - # "following"/"follower" are profile stats at the TOP — not tabs. - if is_tab_intent: - if match.get("y", 0) < screen_height * 0.92: - logger.warning( - f"🛡️ [Structural Guard] VLM hallucinated a navigation tab '{intent}' " - f"in the middle/top of the screen (Y={match.get('y')} | Screen={screen_height}). REJECTING." - ) - return None - - if match.get("y", 0) > screen_height * NAV_BAR_ZONE and not is_nav_intent: - logger.warning( - f"🛡️ [Structural Guard] VLM selected element in nav bar zone for non-nav intent '{intent}': {match['semantic_string']}. REJECTING." - ) - return None - - # ── Structural Guard 3: Already blacklisted ── - if self._is_blacklisted(intent, match["semantic_string"]): - logger.warning( - f"🛡️ [Blacklist Guard] VLM selected previously-rejected element: " - f"'{match['semantic_string']}'. REJECTING." - ) - return None - - logger.info(f"🎯 [Vision Success] VLM identified node {idx} for '{intent}': {match['semantic_string']}") - - # Track but do NOT auto-cache. Wait for confirm_click() from caller. - self._track_click(intent, match) - - return { - "x": match["x"], - "y": match["y"], - "score": 0.85, # Not 1.0 — VLM is provisional, not ground truth - "semantic": f"VLM Match: {match['semantic_string']}", - "source": "agentic_fallback", - "original_attribs": match.get("original_attribs", {}), - } - - except Exception as e: - logger.error(f"[Vision Cortex] Fallback failed: {e}") + # Find grid-like visual nodes + all_nodes = self._parser.get_all_nodes(root) + grid_candidates = [ + n + for n in all_nodes + if not n.text and ("photo" in n.content_desc.lower() or "video" in n.content_desc.lower() or n.area > 50000) + ] + best = self._evaluator.evaluate_grid_visuals(device, persona_interests, grid_candidates) + if best: + return self._translate_node(best) return None - def _ensure_telepathic_agent_context(self): - # Empty placeholder if needed by legacy hooks, or remove if unused. - pass + def evaluate_post_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: + return self._evaluator.evaluate_post_vibe(device, persona_interests) + + def evaluate_profile_vibe(self, device, persona_interests: list[str]) -> Optional[dict]: + return self._evaluator.evaluate_profile_vibe(device, persona_interests) + + def classify_screen_content(self, xml_hierarchy: str, target_class: str) -> Optional[str]: + return self._evaluator.classify_screen_content(xml_hierarchy, target_class) + + def _is_forbidden_action(self, node: dict) -> bool: + """ + Layer 5: Forbids clicks on specific dangerous elements like the quick_capture root container. + """ + res_id = node.get("resource_id", "").lower() + if "quick_capture" in res_id: + return True + return False + + def _get_current_username(self) -> str: + return "" + + def _is_instagram_context(self, xml_string: str) -> bool: + return "com.instagram.android" in xml_string + + def _structural_sanity_check(self, node: dict, intent_description: str, screen_height: int = 2400) -> bool: + """ + Structural guard to ensure nodes are in valid locations for their intents. + """ + intent = intent_description.lower() + y = node.get("y", 0) + semantic = (node.get("semantic_string", "") or "").lower() + + # 1. Navigation Tab Guard (Must be at the bottom) + nav_intents = [ + "tap direct message icon inbox", + "tap inbox", + "tap heart icon notifications", + "tap home tab", + "tap explore tab", + "tap reels tab", + "tap profile tab", + "tap messages tab", + ] + is_nav_intent = any(n in intent for n in nav_intents) + if is_nav_intent: + if y < screen_height * 0.85: + return False + return True + + # 2. Block non-nav intents from clicking in the nav zone + if y >= screen_height * 0.85: + # Not a nav intent, but trying to click the nav bar + return False + + # 3. Post Username Guard + if "post username" in intent: + if "story" in semantic and y < screen_height * 0.2: + # E.g. "Your Story" circle at the top + return False + return True + + # 4. Profile Picture/Story Ring Guard + if "story ring" in intent or "avatar" in intent: + current_user = self._get_current_username() + if current_user and current_user in semantic: + # Don't click our own story + return False + + # 5. Grid Item Guard + if "grid item" in intent: + if y > screen_height * 0.8: # Too low + return False + if "tab" in semantic and "grid" not in semantic: + # Not a grid item + return False + + # 6. Block massive layout containers UNLESS specifically looking for feed/post + MAX_CONTAINER_AREA = 500000 + area = node.get("area", 0) + + is_feed_or_post = "feed" in intent or "post" in intent + is_grid_item = "grid" in intent or "list" in intent + + if area > MAX_CONTAINER_AREA: + # If specifically looking for a grid item, REJECT massive containers (e.g., RecyclerViews) + if is_grid_item: + return False + # Otherwise, only allow if looking for feed/post + if not is_feed_or_post: + return False + + return True diff --git a/pyproject.toml b/pyproject.toml index 5c40165..d5a6964 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ markers = [ "live: tests requiring a live ADB device", "chaos: chaos engineering / corruption tests", "property: hypothesis property-based tests", + "live_llm: tests requiring a live local LLM via Ollama", ] [tool.coverage.run] diff --git a/test_config.yml b/test_config.yml index 0e3a2d3..4495913 100644 --- a/test_config.yml +++ b/test_config.yml @@ -7,7 +7,7 @@ identity: # Unter welchem Account operiert der Bot? username: "marisaundmarc" - + # Wer ist der Bot? (Wichtig für die KI-Kommentare und Profil-Analyse) persona: "Travel blogger, landscape photographer, and outdoors enthusiast" vibe: "friendly, authentic, helpful, and appreciative of good art" @@ -19,14 +19,14 @@ mission: # - stealth_lurker: Liest viel, interagiert aber nur bei extrem hoher Relevanz # - passive_learning: "Dry-Run" Modus. Bot navigiert und lernt, führt aber NIE Aktionen aus. strategy: "aggressive_growth" - + # Wie kritisch ist der Bot bei fremden Posts? (Hoch = nur Meisterwerke, Niedrig = fast alles) - selectivity_threshold: "high" - + selectivity_threshold: "high" + # Wen sucht der Bot? (Alias für target-audience) target_audience: "travel, landscape, nature, mountain photography, wanderlust" # persona_interests: "travel, landscape, nature" # Alternative zu target_audience - + # Was hasst der Bot absolut? (Sofortiger Skip) blacklist_topics: "onlyfans, nsfw, sale, discount, promo, 18+, giveaway, crypto" @@ -46,17 +46,17 @@ interactions: comment_percentage: 40 # Moderater Wert, da Kommentare "dry" sind follow_percentage: 100 # IMMER folgen, wenn das Profil als relevant bewertet wurde stories_percentage: 100 # IMMER Stories schauen, um menschlich zu wirken - + # Detail-Limits pro Profil/Post likes_count: "2-3" # 2-3 schnelle Likes auf dem Profil hinterlassen (sehr starkes Signal) stories_count: "1-2" # 1-2 Stories anschauen (sehr menschliches Verhalten) # Comment Dry Run: Wenn true, überlegt sich die AI geniale Kommentare, postet sie aber nicht in echt. dry_run_comments: true - + # Wahrscheinlichkeit (in Prozent), fremde Profile VOR dem Kommentieren tiefgründig zu analysieren profile_learning_percentage: 100 # IMMER Profile analysieren -> Trigger für den Follow/Like Flow - + # Wahrscheinlichkeit (in Prozent), das Bild visuell zu analysieren (Screenshot -> LLM), bevor interagiert wird visual_vibe_check_percentage: 100 @@ -76,7 +76,7 @@ limits: daily_budget_hours: 2.5 # working_hours: "09:00-21:00" # In welchem Fenster der Bot laufen darf # time_delta_session: "60-120" # Minuten Pause zwischen Sessions - + # Absolute Sicherheitsnetze pro Tag/Lauf max_comments_per_day: 40 # total_likes_limit: 300 @@ -97,7 +97,7 @@ limits: ignore_close_friends: true # Ignoriere alles (Posts/Stories) von "Enge Freunde" # ── Infrastructure & System (Nur für Entwickler) ── -device: 192.168.1.206:34201 +device: 192.168.1.206:42171 app-id: com.instagram.android debug: true diff --git a/tests/anomalies/test_hardware_anomalies.py b/tests/anomalies/test_hardware_anomalies.py index d1153b9..575ff33 100644 --- a/tests/anomalies/test_hardware_anomalies.py +++ b/tests/anomalies/test_hardware_anomalies.py @@ -1,9 +1,10 @@ -import pytest import os import time from unittest.mock import MagicMock, patch -from GramAddict.core.bot_flow import _wait_for_post_loaded, _run_zero_latency_feed_loop, FEED_MARKERS -from GramAddict.core.device_facade import DeviceFacade + +import pytest + +from GramAddict.core.bot_flow import FEED_MARKERS, _run_zero_latency_feed_loop, _wait_for_post_loaded ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DUMPS = { @@ -12,14 +13,17 @@ DUMPS = { } FIXTURE_DIR = os.path.join(ROOT_DIR, "fixtures") + def mutate_xml_to_foreign(xml_content: str) -> str: """Removes meaningful text content to simulate a language failure or empty state.""" import re + # Strip text and content-desc xml = re.sub(r'text="[^"]*"', 'text=""', xml_content) xml = re.sub(r'content-desc="[^"]*"', 'content-desc=""', xml) return xml + def mutate_xml_remove_feed_markers(xml_content: str) -> str: """Removes all feed markers to simulate a grid view or random popup.""" xml = xml_content @@ -27,21 +31,26 @@ def mutate_xml_remove_feed_markers(xml_content: str) -> str: xml = xml.replace(marker, "some_random_id") return xml + class ConfigMock: def __init__(self): self.args = MagicMock() self.args.interact_percentage = 0 self.args.comment_percentage = 0 + @pytest.fixture def test_dumps(): dumps = {} with open(DUMPS["organic"], "r") as f: dumps["post"] = f.read() # Fake explore grid that lacks ALL feed markers - dumps["grid"] = '' + dumps["grid"] = ( + '' + ) return dumps + def test_slow_loading_post_recovery(test_dumps): """ Test that _wait_for_post_loaded correctly handles a delay where the @@ -50,33 +59,35 @@ def test_slow_loading_post_recovery(test_dumps): device = MagicMock() # Simulate: Grid -> Grid -> Error -> Post device.dump_hierarchy.side_effect = [ - test_dumps["grid"], + test_dumps["grid"], test_dumps["grid"], Exception("uiautomator2 temp failure"), - test_dumps["post"] + test_dumps["post"], ] - + # We patch sleep to make the test super fast - with patch('GramAddict.core.bot_flow.sleep', return_value=None): + with patch("GramAddict.core.bot_flow.sleep", return_value=None): start = time.time() success = _wait_for_post_loaded(device, timeout=5) # Should return true when it hits the 4th element assert success is True assert device.dump_hierarchy.call_count == 4 + def test_wait_timeout_aborts_gracefully(test_dumps): """Test what happens if the network is so slow it times out entirely.""" device = MagicMock() # Always return grid device.dump_hierarchy.return_value = test_dumps["grid"] - + # Patch time.time to simulate 6 seconds passing immediately # We add sequence padding because python's logger internally uses time.time() - with patch('GramAddict.core.bot_flow.time.time', side_effect=[0, 1, 6, 6, 6, 6, 6, 6, 6, 6]): - with patch('GramAddict.core.bot_flow.sleep', return_value=None): + with patch("time.time", side_effect=[0, 1, 6, 6, 6, 6, 6, 6, 6, 6]): + with patch("GramAddict.core.bot_flow.sleep", return_value=None): success = _wait_for_post_loaded(device, timeout=5) assert success is False + def test_empty_content_extraction_guard(test_dumps): """ Test that if a post is loaded, but it has strange empty text (foreign language or bug), @@ -85,38 +96,47 @@ def test_empty_content_extraction_guard(test_dumps): device = MagicMock() nav_graph = MagicMock() configs = ConfigMock() - + # We create a fake active inference engine to just break the loop after 1 iteration ai = MagicMock() # Dopamine engine controls loop exit dopamine = MagicMock() - dopamine.is_app_session_over.side_effect = [False, True] # Run once, then exit + dopamine.is_app_session_over.side_effect = [False, True] # Run once, then exit dopamine.wants_to_change_feed.return_value = False dopamine.wants_to_doomscroll.return_value = False - + cognitive_stack = { "dopamine": dopamine, "active_inference": ai, - "resonance": None, "growth_brain": None, "swarm": None, "darwin": None + "resonance": None, + "growth_brain": None, + "swarm": None, + "darwin": None, } - + # Mutate the post so it has NO text or description broken_xml = mutate_xml_to_foreign(test_dumps["post"]) device.dump_hierarchy.return_value = broken_xml - + from GramAddict.core.situational_awareness import SituationType - with patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \ - patch('GramAddict.core.bot_flow.sleep'), \ - patch('GramAddict.core.situational_awareness.SituationalAwarenessEngine.perceive', return_value=SituationType.NORMAL): - + + with ( + patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, + patch("GramAddict.core.bot_flow.sleep"), + patch( + "GramAddict.core.situational_awareness.SituationalAwarenessEngine.perceive", + return_value=SituationType.NORMAL, + ), + ): result = _run_zero_latency_feed_loop(device, None, nav_graph, configs, MagicMock(), "HomeFeed", cognitive_stack) - + # Ensure scroll was called (the recovery mechanism) assert mock_scroll.called # Check that we never called resonance evaluation because we broke early assert not ai.predict_state.called assert result == "FEED_EXHAUSTED" + def test_missing_feed_markers_guard(test_dumps): """ Test that if the UI is completely foreign (e.g., a system popup), @@ -124,23 +144,23 @@ def test_missing_feed_markers_guard(test_dumps): """ device = MagicMock() configs = ConfigMock() - + dopamine = MagicMock() dopamine.is_app_session_over.side_effect = [False, True] dopamine.wants_to_change_feed.return_value = False dopamine.wants_to_doomscroll.return_value = False - + cognitive_stack = {"dopamine": dopamine, "growth_brain": None, "active_inference": None} - + # Mutate XML to remove all FEED MARKERS alien_xml = mutate_xml_remove_feed_markers(test_dumps["post"]) device.dump_hierarchy.return_value = alien_xml - - with patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \ - patch('GramAddict.core.bot_flow.sleep'): + + with patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, patch("GramAddict.core.bot_flow.sleep"): _run_zero_latency_feed_loop(device, None, MagicMock(), configs, MagicMock(), "HomeFeed", cognitive_stack) - -@patch('GramAddict.core.device_facade.u2') + + +@patch("GramAddict.core.device_facade.u2") def test_xpath_watcher_initialization(mock_u2): """ Test fixing the critical watcher API bug. @@ -148,21 +168,22 @@ def test_xpath_watcher_initialization(mock_u2): """ mock_d = MagicMock() mock_u2.connect.return_value = mock_d - + # Setup mock chain: deviceV2.watcher("crash_dialog").when(...) mock_watcher = MagicMock() mock_d.watcher.return_value = mock_watcher mock_when = MagicMock() mock_watcher.when.return_value = mock_when - + # Just init the facade from GramAddict.core.device_facade import create_device + device = create_device("fake_serial", "com.fake.app", MagicMock()) - + # Verify exact API call structure for XPath mock_d.watcher.assert_any_call("crash_dialog") mock_d.watcher.assert_any_call("system_dialog") - + # We can't perfectly assert the chained arguments natively without a bit of inspection, # but we can verify it didn't crash and called start assert mock_d.watcher.start.called diff --git a/tests/anomalies/test_telepathic_guards.py b/tests/anomalies/test_telepathic_guards.py new file mode 100644 index 0000000..a570f23 --- /dev/null +++ b/tests/anomalies/test_telepathic_guards.py @@ -0,0 +1,69 @@ +from GramAddict.core.telepathic_engine import TelepathicEngine + + +class TestTelepathicGuards: + def setup_method(self): + self.engine = TelepathicEngine() + + def test_strict_story_ring_guard(self): + """ + TDD: Story rings MUST be physically near the top of the screen (y < 30%). + Post profile headers that appear further down must be aggressively blocked + when the intent is 'tap story ring avatar'. + """ + intent = "tap story ring avatar" + screen_height = 2400 + + # Valid Story Ring (Top of screen, but below status bar) + valid_story = {"resource_id": "reel_ring", "y": 300, "area": 100} + assert self.engine._structural_sanity_check(valid_story, intent, screen_height) is True + + # Invalid Story Ring (Hallucination: Post profile header in the feed) + invalid_story = {"resource_id": "row_feed_profile_header", "y": 800, "area": 100} + assert self.engine._structural_sanity_check(invalid_story, intent, screen_height) is False + + def test_strict_button_guard(self): + """ + TDD: When explicitly looking for a 'button', nodes that declare themselves + as profiles (e.g. 'go to profile') must be blocked, to prevent accidental + profile visits when clicking 'like'. + """ + intent = "Heart like button for comment" + screen_height = 2400 + + # Valid Like Button + valid_btn = {"resource_id": "like_button", "semantic_string": "Like", "y": 1000, "area": 100} + assert self.engine._structural_sanity_check(valid_btn, intent, screen_height) is True + + # Invalid Profile Link masquerading as a match due to string proximity + invalid_prof = { + "resource_id": "username", + "semantic_string": "Go to cayleighanddavid's profile", + "y": 1000, + "area": 100, + } + assert self.engine._structural_sanity_check(invalid_prof, intent, screen_height) is False + + # However, if the intent *is* profile, it should pass + intent_prof = "go to profile" + assert self.engine._structural_sanity_check(invalid_prof, intent_prof, screen_height) is True + + def test_like_semantic_verification(self): + """ + TDD: Verify that 'unlike' is treated as a successful 'Like' action, + because tapping 'Like' changes the state to 'Unlike' in English Instagram. + """ + # Testing the specific regex logic inside verify_success + import re + + xml_dump_success = '' + intent = "tap like button" + + marker_found = re.search(r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_success.lower()) + assert marker_found is not None + + xml_dump_fail = '' + marker_found_fail = re.search( + r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_fail.lower() + ) + assert marker_found_fail is None diff --git a/tests/conftest.py b/tests/conftest.py index da4c74e..a5e6618 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,130 +1,169 @@ -import pytest import logging import os from unittest.mock import MagicMock +import pytest + + def pytest_addoption(parser): parser.addoption( - "--live", action="store_true", default=False, help="run tests against a live ADB device (disable DeviceFacade mocks)" + "--live", + action="store_true", + default=False, + help="run tests against a live ADB device (disable DeviceFacade mocks)", ) + MagicMock.app_id = "com.instagram.android" MagicMock._get_current_app = MagicMock(return_value="com.instagram.android") + class MockArgs: def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) + class MockConfigs: def __init__(self, args): self.args = args -from unittest.mock import create_autospec, MagicMock + +from unittest.mock import MagicMock, create_autospec + from GramAddict.core.device_facade import DeviceFacade from GramAddict.core.telepathic_engine import TelepathicEngine + def create_mock_device(): mock = create_autospec(DeviceFacade, instance=True) mock.app_id = "com.instagram.android" mock.device_id = "test_device" - + mock.info = {"displayWidth": 1080, "displayHeight": 2400} mock.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400} mock.cm_to_pixels.side_effect = lambda cm: int(cm * 10) mock.shell.return_value = "" # Ensure SendEventInjector detection gets a string import uuid - mock.dump_hierarchy.side_effect = lambda: f"" - + + mock.dump_hierarchy.side_effect = ( + lambda: f'' + ) + return mock + def create_mock_telepathic_engine(): mock = create_autospec(TelepathicEngine, instance=True) mock.find_best_node.return_value = {"x": 500, "y": 500, "confidence": 0.9} - mock.evaluate_profile_vibe.return_value = {"quality_score": 8, "matches_niche": True, "reason": "Mocked positive vibe"} - mock.evaluate_grid_visuals.return_value = {"x": 500, "y": 500, "score": 0.99, "semantic": "Mocked matching grid cell", "source": "vlm_grid"} - mock._extract_semantic_nodes.return_value = [{"x": 500, "y": 500, "semantic_string": "dummy node"}] + mock.evaluate_profile_vibe.return_value = { + "quality_score": 8, + "matches_niche": True, + "reason": "Mocked positive vibe", + } + mock.evaluate_grid_visuals.return_value = { + "x": 500, + "y": 500, + "score": 0.99, + "semantic": "Mocked matching grid cell", + "source": "vlm_grid", + } + mock.find_best_node.return_value = {"x": 500, "y": 500, "semantic_string": "dummy node"} return mock + @pytest.fixture def mock_logger(): return logging.getLogger("test") + @pytest.fixture def device(request): if request.config.getoption("--live"): - from GramAddict.core.device_facade import create_device - import yaml import os - + + import yaml + + from GramAddict.core.device_facade import create_device + device_id = "emulator-5554" app_id = "com.instagram.android" - + config_path = "test_config.yml" if os.path.exists(config_path): try: - with open(config_path, 'r', encoding='utf-8') as f: + with open(config_path, "r", encoding="utf-8") as f: config = yaml.safe_load(f) if config: device_id = config.get("device", device_id) app_id = config.get("app-id", app_id) except Exception as e: print(f"⚠️ Warning: Could not load {config_path}: {e}") - + print(f"🚀 Connecting to live device: {device_id} (App: {app_id})") return create_device(device_id, app_id) return create_mock_device() + @pytest.fixture(autouse=True) def reset_singletons(): """Ensure all core engine singletons are fresh for each test.""" - from GramAddict.core.telepathic_engine import TelepathicEngine - from GramAddict.core.goap import GoalExecutor - from GramAddict.core.situational_awareness import SituationalAwarenessEngine - - from GramAddict.core.qdrant_memory import QdrantBase from GramAddict.core.behaviors import PluginRegistry + from GramAddict.core.goap import GoalExecutor from GramAddict.core.physics.biomechanics import PhysicsBody from GramAddict.core.physics.sendevent_injector import SendEventInjector - + from GramAddict.core.qdrant_memory import QdrantBase + from GramAddict.core.situational_awareness import SituationalAwarenessEngine + from GramAddict.core.telepathic_engine import TelepathicEngine + TelepathicEngine.reset() GoalExecutor.reset() SituationalAwarenessEngine.reset() PluginRegistry.reset() PhysicsBody.reset() SendEventInjector.reset() - + QdrantBase._connection_failed_logged = False - + from GramAddict.core.dojo_engine import DojoEngine + if hasattr(DojoEngine, "reset"): DojoEngine.reset() else: DojoEngine._instance = None - + # Aggressively wipe on-disk session files to prevent state leakage in tests - for f in ["telepathic_memory.json", "telepathic_blacklist.json", "growth_brain_memory.json", "gramaddict_nav_map.json", "l2_channels_cache.json"]: + for f in [ + "telepathic_memory.json", + "telepathic_blacklist.json", + "growth_brain_memory.json", + "gramaddict_nav_map.json", + "l2_channels_cache.json", + ]: if os.path.exists(f): try: os.remove(f) except Exception: pass yield - + # Post-test cleanup PhysicsBody.reset() SendEventInjector.reset() + @pytest.fixture(autouse=True) def telepathic_mock(monkeypatch, request): if request.config.getoption("--live"): # TelepathicEngine is a singleton, allow it to run natively return None import GramAddict.core.telepathic_engine + engine = create_mock_telepathic_engine() monkeypatch.setattr(GramAddict.core.telepathic_engine.TelepathicEngine, "get_instance", lambda: engine) return engine - + + @pytest.fixture def mock_cognitive_stack(): stack = { @@ -138,7 +177,7 @@ def mock_cognitive_stack(): "nav_graph": MagicMock(), "zero_engine": MagicMock(), "crm": MagicMock(), - "telepathic": create_mock_telepathic_engine() + "telepathic": create_mock_telepathic_engine(), } stack["radome"].sanitize_xml.side_effect = lambda x: x return stack diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e2d7fe1..127a5ec 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,11 +1,13 @@ -import sys import os -import pytest +import sys import time from unittest.mock import MagicMock + +import pytest + from GramAddict.core import utils -# Force Qdrant mocking globally across ALL E2E tests so we never +# Force Qdrant mocking globally across ALL E2E tests so we never # block on connection refused trying to hit localhost:6344 mock_qdrant = MagicMock() @@ -16,11 +18,12 @@ mock_qdrant.get_collection.return_value = mock_collection sys.modules["qdrant_client"].QdrantClient = MagicMock(return_value=mock_qdrant) + @pytest.fixture def e2e_device_dump_injector(request): """ Provides a factory to mock device.dump_hierarchy using real XML files. - Will gracefully fail with a comprehensive assertion if the file is missing + Will gracefully fail with a comprehensive assertion if the file is missing (per 'ECHTE DUMPS fehlen' reporting requirement). """ if request.config.getoption("--live"): @@ -29,30 +32,36 @@ def e2e_device_dump_injector(request): def _inject_dump(device_mock, xml_filename): fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") xml_path = os.path.join(fix_dir, xml_filename) - + if not os.path.exists(xml_path): - pytest.fail(f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.", pytrace=False) - + pytest.fail( + f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.", + pytrace=False, + ) + with open(xml_path, "r") as f: real_xml = f.read() - + device_mock.dump_hierarchy.return_value = real_xml return real_xml - + return _inject_dump + class VirtualClock: def __init__(self): self.time = 0.0 self.animation_target_time = 0.0 - + def sleep(self, seconds): - if hasattr(seconds, '__iter__'): - return # For edge case where something weird is passed + if hasattr(seconds, "__iter__"): + return # For edge case where something weird is passed self.time += float(seconds) + clock = VirtualClock() + @pytest.fixture def dynamic_e2e_dump_injector(monkeypatch, request): """ @@ -66,9 +75,9 @@ def dynamic_e2e_dump_injector(monkeypatch, request): def _inject(device_mock, state_map, initial_xml): from GramAddict.core.q_nav_graph import QNavGraph - + fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") - + def load_xml(filename): path = os.path.join(fix_dir, filename) if not os.path.exists(path): @@ -79,47 +88,56 @@ def dynamic_e2e_dump_injector(monkeypatch, request): # History stack to allow "back" navigation device_mock._xml_history = [load_xml(initial_xml)] device_mock._current_active_xml = device_mock._xml_history[-1] - + import uuid + def _dump_hierarchy_hook(): if clock.time < clock.animation_target_time: - pytest.fail(f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! " - f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. " - f"Add a time.sleep() guard before interacting with the UI after a click.", pytrace=False) + pytest.fail( + f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! " + f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. " + f"Add a time.sleep() guard before interacting with the UI after a click.", + pytrace=False, + ) xml = device_mock._current_active_xml if xml and "" in xml: - xml = xml.replace("", f"") + xml = xml.replace("", f'') return xml device_mock.dump_hierarchy.side_effect = _dump_hierarchy_hook - + def _press_hook(key, *args, **kwargs): if key == "back" and len(device_mock._xml_history) > 1: device_mock._xml_history.pop() device_mock._current_active_xml = device_mock._xml_history[-1] clock.animation_target_time = clock.time + 1.5 + device_mock.press.side_effect = _press_hook - + class DummyEngine: def find_best_node(self, *args, **kwargs): return {"x": 500, "y": 500, "skip": False, "score": 1.0, "source": "e2e_mock"} + def verify_success(self, *args, **kwargs): return True + def confirm_click(self, *args, **kwargs): pass + def reject_click(self, *args, **kwargs): pass - + original_execute = QNavGraph._execute_transition from GramAddict.core.goap import GoalExecutor + original_goap_execute = GoalExecutor._execute_action - + def _mock_execute_transition(nav_self, action, zero_engine=None, max_retries=2): - if action == 'tap_post_username': + if action == "tap_post_username": return True - + original_click = nav_self.device.click - + def _click_hook(obj=None, *args, **kwargs): original_click(obj, *args, **kwargs) if action in state_map: @@ -127,22 +145,24 @@ def dynamic_e2e_dump_injector(monkeypatch, request): device_mock._xml_history.append(new_xml) device_mock._current_active_xml = new_xml clock.animation_target_time = clock.time + 1.5 - + nav_self.device.click = _click_hook - + try: - success = original_execute(nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries) + success = original_execute( + nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries + ) return success finally: nav_self.device.click = original_click def _mock_execute_action(goap_self, action, goal=None): action_key = action.replace(" ", "_") - if action_key == 'tap_post_username': + if action_key == "tap_post_username": return True - + original_click = goap_self.device.click - + def _click_hook(obj=None, *args, **kwargs): original_click(obj, *args, **kwargs) if action_key in state_map: @@ -155,20 +175,21 @@ def dynamic_e2e_dump_injector(monkeypatch, request): device_mock._xml_history.append(new_xml) device_mock._current_active_xml = new_xml clock.animation_target_time = clock.time + 1.5 - + goap_self.device.click = _click_hook - + try: success = original_goap_execute(goap_self, action, goal=goal) return success finally: goap_self.device.click = original_click - + monkeypatch.setattr(QNavGraph, "_execute_transition", _mock_execute_transition) monkeypatch.setattr(GoalExecutor, "_execute_action", _mock_execute_action) - + return _inject + @pytest.fixture(autouse=True) def mock_all_delays(monkeypatch, request): """ @@ -180,49 +201,72 @@ def mock_all_delays(monkeypatch, request): return global clock - clock.time = 0.0 # reset for test + clock.time = 0.0 # reset for test clock.animation_target_time = 0.0 - + def simulate_sleep(seconds): clock.sleep(seconds) money_sleep = lambda x: simulate_sleep(x) - random_sleep = lambda *args, **kwargs: simulate_sleep(1.0) # Assume 1.0 minimum for randoms - + random_sleep = lambda a=1.0, b=2.0, *args, **kwargs: simulate_sleep(max(1.5, float(a))) + monkeypatch.setattr(time, "sleep", money_sleep) monkeypatch.setattr(utils, "random_sleep", random_sleep) monkeypatch.setattr(utils, "sleep", money_sleep) - + # Needs to capture specific module sleeps depending on how they imported it try: from GramAddict.core import bot_flow + monkeypatch.setattr(bot_flow, "sleep", money_sleep) - monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound - + monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound + if hasattr(bot_flow, "random_sleep"): + monkeypatch.setattr(bot_flow, "random_sleep", random_sleep) + from GramAddict.core import q_nav_graph + monkeypatch.setattr(q_nav_graph.random, "uniform", lambda a, b: float(a)) - + if hasattr(q_nav_graph, "random_sleep"): + monkeypatch.setattr(q_nav_graph, "random_sleep", random_sleep) + + from GramAddict.core import goap + + if hasattr(goap, "random"): + monkeypatch.setattr(goap.random, "uniform", lambda a, b: float(a)) + if hasattr(goap, "random_sleep"): + monkeypatch.setattr(goap, "random_sleep", random_sleep) + + monkeypatch.setattr(utils.random, "uniform", lambda a, b: float(a)) + from GramAddict.core import device_facade + monkeypatch.setattr(device_facade, "sleep", money_sleep) monkeypatch.setattr(device_facade.random, "uniform", lambda a, b: float(a)) - except Exception: - pass - + if hasattr(device_facade, "random_sleep"): + monkeypatch.setattr(device_facade, "random_sleep", random_sleep) + except Exception as e: + print(f"Mocking delays exception: {e}") + # Standardize DarwinEngine across tests to prevent mockup math errors on session end try: from GramAddict.core.darwin_engine import DarwinEngine + monkeypatch.setattr(DarwinEngine, "evaluate_session_end", lambda *args, **kwargs: None) except ImportError: pass + @pytest.fixture(autouse=True) def mock_identity_guard(monkeypatch): import GramAddict.core.bot_flow + monkeypatch.setattr(GramAddict.core.bot_flow, "verify_and_switch_account", lambda *args, **kwargs: True) + @pytest.fixture def e2e_configs(): import argparse + configs = MagicMock() configs.username = "testuser" configs.args = argparse.Namespace( @@ -254,6 +298,7 @@ def e2e_configs(): ) return configs + @pytest.fixture(autouse=True) def mock_sae_perceive(request, monkeypatch): """ @@ -263,9 +308,15 @@ def mock_sae_perceive(request, monkeypatch): """ if "test_e2e_sae.py" in str(request.node.fspath): return + if "test_e2e_real_llm_learning.py" in str(request.node.fspath): + return if request.config.getoption("--live"): return - - import GramAddict.core.situational_awareness - monkeypatch.setattr(GramAddict.core.situational_awareness.SituationalAwarenessEngine, "perceive", lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL) + import GramAddict.core.situational_awareness + + monkeypatch.setattr( + GramAddict.core.situational_awareness.SituationalAwarenessEngine, + "perceive", + lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL, + ) diff --git a/tests/e2e/test_e2e_config_goal_limits.py b/tests/e2e/test_e2e_config_goal_limits.py new file mode 100644 index 0000000..899e989 --- /dev/null +++ b/tests/e2e/test_e2e_config_goal_limits.py @@ -0,0 +1,92 @@ +from unittest.mock import MagicMock, patch + +from GramAddict.core.bot_flow import _run_zero_latency_feed_loop +from GramAddict.core.session_state import SessionState + + +def test_feed_loop_respects_config_limits(device, mock_cognitive_stack): + """ + Testet, ob die Config (Ziele/Limits) beachtet wird: + Erreicht der Bot sein Ziel (z.B. total_likes_limit) und stoppt er dann? + """ + + # 1. Simulate dopamine so we don't naturally exit early due to session time + mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False + mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False + mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False + mock_cognitive_stack[ + "resonance" + ].calculate_resonance.return_value = 0.75 # < 0.8 to avoid rabbit hole, but high enough to engage + + # 2. Setup Config mimicking test_config.yml goals + configs = MagicMock() + configs.args.total_likes_limit = 2 + configs.args.end_if_likes_limit_reached = True + configs.args.interact_percentage = 100 + configs.args.likes_percentage = 100 + configs.args.follow_percentage = 0 + configs.args.comment_percentage = 0 + configs.args.visual_vibe_check_percentage = 0 + configs.args.profile_learning_percentage = 0 + configs.args.repost_percentage = 0 + + # 3. Setup real SessionState to track limits correctly based on config + session_state = SessionState(configs) + session_state.set_limits_session() + + # 4. Provide a UI dump that has content so the bot interacts + device.dump_hierarchy.return_value = """ + + + + + """ + + # Prevent radome from stripping our mock structure + mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x + mock_cognitive_stack["nav_graph"].do.return_value = True + + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract, + patch("GramAddict.core.bot_flow._align_active_post", return_value=False), + patch("GramAddict.core.bot_flow._humanized_scroll"), + patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}), + patch("GramAddict.core.bot_flow._humanized_click") as mock_click, + patch("GramAddict.core.bot_flow.sleep"), + patch("GramAddict.core.bot_flow.random.random", return_value=0.1), + ): # Force pass probabilities + mock_extract.return_value = {"username": "test_user", "description": "test image", "caption": ""} + + mock_instance = MockTelepathic.get_instance.return_value + # Nodes for standard flow + mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}] + # When finding the like button + mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False} + + mock_cognitive_stack["telepathic"] = mock_instance + + # We'll patch `_humanized_click` to increment the like counter to simulate the interaction succeeding. + def mock_click_side_effect(*args, **kwargs): + session_state.totalLikes += 1 + session_state.add_interaction("test_user", succeed=True, followed=False, scraped=False) + + mock_click.side_effect = mock_click_side_effect + + # Run the autonomous loop + result = _run_zero_latency_feed_loop( + device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + + # 5. Verify expectations + # The loop should break when `totalLikes` reaches at least 2 (total_likes_limit) + assert session_state.totalLikes >= 2, f"Expected at least 2 likes, got {session_state.totalLikes}" + + # Loop terminates cleanly because of limit + assert result == "FEED_EXHAUSTED", "Der Feed-Loop sollte durch das Limit-Breakout terminieren!" diff --git a/tests/e2e/test_e2e_navigation_escape_dm_trap.py b/tests/e2e/test_e2e_navigation_escape_dm_trap.py index 014d772..f5f2581 100644 --- a/tests/e2e/test_e2e_navigation_escape_dm_trap.py +++ b/tests/e2e/test_e2e_navigation_escape_dm_trap.py @@ -42,9 +42,11 @@ Expected Behaviour After Green Phase 3. ``TelepathicEngine.find_best_node()`` with a profile-grid intent returns ``None`` (or a ``{"blocked_by_dm_thread": True}`` sentinel) when the XML is a DM thread. """ + import os +from unittest.mock import MagicMock + import pytest -from unittest.mock import MagicMock, patch, PropertyMock # ────────────────────────────────────────────── # Fixture Helpers @@ -65,14 +67,12 @@ def _load_fixture(filename: str) -> str: return f.read() - - - # ────────────────────────────────────────────── # Test 3: Structural Guard — TelepathicEngine must refuse to find # profile-intent nodes inside a DM thread # ────────────────────────────────────────────── + class TestTelepathicEngineDmForbiddenZone: """ RED: When the visible XML is a DM thread and the intent is profile-related @@ -86,25 +86,15 @@ class TestTelepathicEngineDmForbiddenZone: """ def _make_engine(self): - with patch("GramAddict.core.telepathic_engine.QdrantBase") as MockQdrant, \ - patch("GramAddict.core.telepathic_engine.query_telepathic_llm"), \ - patch("GramAddict.core.telepathic_engine.dump_ui_state"): - from GramAddict.core.telepathic_engine import TelepathicEngine - e = TelepathicEngine.__new__(TelepathicEngine) - e._embedding_cache = {} - e._intent_cache = {} - e._blacklist = {} - e._memory = {} - e._cached_username = "testuser" - e._cached_app_id = "com.instagram.android" - # Mock embedding_helper so vector stage is a no-op (returns None → falls to VLM) - mock_helper = MagicMock() - mock_helper._get_embedding.return_value = None - e.embedding_helper = mock_helper - - # Mock ui_memory so Qdrant Fast Paths don't crash - e.ui_memory = MagicMock() - e.ui_memory.retrieve_memory.return_value = None + # We only need a raw TelepathicEngine instance + from GramAddict.core.telepathic_engine import TelepathicEngine + + TelepathicEngine._instance = None + e = TelepathicEngine() + + # Mock the internal resolver's LLM call to prevent actual OLLAMA requests during fast-paths + e._resolver.resolve = MagicMock(return_value=None) + return e def test_profile_intent_is_blocked_when_dm_thread_is_active(self): @@ -128,10 +118,7 @@ class TestTelepathicEngineDmForbiddenZone: ] for intent in profile_seeking_intents: - # Patch embedding to None so vector stage is a no-op; VLM path also mocked off - with patch.object(engine, "_get_cached_embedding", return_value=None), \ - patch.object(engine, "_vision_cortex_fallback", return_value=None): - result = engine.find_best_node(dm_xml, intent, device=device) + result = engine.find_best_node(dm_xml, intent, device=device) # The keyword fast-path WILL find nodes in the DM thread (e.g. the 'view_profile_button' # has 'profile' in its resource-id, matching the intent). The guard must intercept @@ -162,10 +149,7 @@ class TestTelepathicEngineDmForbiddenZone: # This intent is used by dm_engine.py to find the message composer dm_intent = "find the message input text field" - # Mock the embedding calls so we don't block on Qdrant during unit test - with patch.object(engine, "_get_cached_embedding", return_value=None), \ - patch.object(engine, "_vision_cortex_fallback", return_value=None): - result = engine.find_best_node(dm_xml, dm_intent, device=device) + result = engine.find_best_node(dm_xml, dm_intent, device=device) # Should NOT be blocked — DM intents are valid inside a DM thread # (may be None if keyword/vector stage misses, but must NOT be blocked_by_dm_thread) @@ -174,4 +158,3 @@ class TestTelepathicEngineDmForbiddenZone: f"DM intent '{dm_intent}' was incorrectly blocked inside a DM thread. " f"The structural guard must only block PROFILE-seeking intents." ) - diff --git a/tests/e2e/test_e2e_real_llm_learning.py b/tests/e2e/test_e2e_real_llm_learning.py new file mode 100644 index 0000000..3c643c6 --- /dev/null +++ b/tests/e2e/test_e2e_real_llm_learning.py @@ -0,0 +1,163 @@ +""" +Real LLM + Qdrant Integration Test +Tests the extreme learning behavior of the autonomous engine by hitting +the real local Ollama instance and storing/retrieving from local Qdrant. + +Requirements: +- Ollama must be running on localhost:11434 +- llama3.2-vision must be available locally +- Qdrant must be running locally +""" + +import time +import uuid +from unittest.mock import MagicMock, patch + +import pytest + +from GramAddict.core.device_facade import DeviceFacade +from GramAddict.core.qdrant_memory import ScreenMemoryDB +from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType + +# ───────────────────────────────────────────────────── +# Test Setup & Isolation +# ───────────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def isolated_screen_memory(): + """Ensures we use a separate Qdrant collection for real LLM testing and clean it.""" + # We patch __init__ so that any instantiation uses the test collection + original_init = ScreenMemoryDB.__init__ + + def test_init(self): + super(ScreenMemoryDB, self).__init__(collection_name="test_real_llm_screens") + + ScreenMemoryDB.__init__ = test_init + + db = ScreenMemoryDB() + if db.is_connected: + db.wipe_collection() + + yield db + + # Restore original + ScreenMemoryDB.__init__ = original_init + + +def make_mock_device(app_id="com.instagram.android"): + device = MagicMock(spec=DeviceFacade) + device.app_id = app_id + device.deviceV2 = MagicMock() + device.dump_hierarchy = MagicMock() + device.click = MagicMock() + device.press = MagicMock() + device.app_start = MagicMock() + device._trace_counter = 0 + device._trace_dir = "/tmp/test_traces" + return device + + +# ───────────────────────────────────────────────────── +# Tests +# ───────────────────────────────────────────────────── + + +@pytest.mark.live_llm +def test_real_llm_learning_and_unlearning(isolated_screen_memory): + """ + Testet das echte Lernverhalten: + 1. Pass: Unbekanntes XML -> LLM wird angefragt -> Speichert in Qdrant + 2. Pass: Gleiches XML -> LLM wird NICHT angefragt -> Holt aus Qdrant + 3. Pass (Unlearn): Wir löschen den State (Simulation Fehler) -> Gleiches XML -> LLM wird wieder angefragt + """ + + # Check if Qdrant is connected. If not, we skip the test gracefully. + if not isolated_screen_memory.is_connected: + pytest.skip("Qdrant is not running locally. Skipping live integration test.") + + # Generate completely unique XML so it's guaranteed NOT in any cache + random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}" + random_text = f"REAL_LLM_TEST_{uuid.uuid4().hex[:8]}" + + # A simple modal to trigger perception + chaos_xml = f""" + + + + + + + +""" + + device = make_mock_device() + sae = SituationalAwarenessEngine(device) + + # We patch the underlying LLM call just to spy on it (wraps the original function) + from GramAddict.core.llm_provider import query_telepathic_llm + + with patch("GramAddict.core.llm_provider.query_telepathic_llm", wraps=query_telepathic_llm) as spy_llm: + # --------------------------------------------------------- + # PASS 1: The Initial Encounter (Learn) + # --------------------------------------------------------- + print(f"\n--- PASS 1: Querying real LLM for '{random_text}' ---") + start_time = time.time() + + # This will block and hit the real local Ollama + result_pass1 = sae.perceive(chaos_xml) + + duration = time.time() - start_time + print(f"Pass 1 completed in {duration:.2f}s. Result: {result_pass1}") + + # Assertions + assert spy_llm.call_count == 1, "LLM was not called on unknown XML!" + assert result_pass1 in [ + SituationType.OBSTACLE_MODAL, + SituationType.NORMAL, + SituationType.OBSTACLE_FOREIGN_APP, + ], "Invalid LLM perception result" + + spy_llm.reset_mock() + + # Give Qdrant a split second to index the new point + time.sleep(0.5) + + # --------------------------------------------------------- + # PASS 2: The Recall (Cache Hit) + # --------------------------------------------------------- + print("\\n--- PASS 2: Recalling from Qdrant ---") + start_time = time.time() + + result_pass2 = sae.perceive(chaos_xml) + + duration = time.time() - start_time + print(f"Pass 2 completed in {duration:.2f}s. Result: {result_pass2}") + + # Assertions + assert spy_llm.call_count == 0, "LLM was called again despite being in Qdrant!" + assert result_pass2 == result_pass1, "Qdrant cache returned a different result than the initial LLM call!" + assert duration < 1.0, f"Qdrant retrieval took too long ({duration:.2f}s). Should be sub-second." + + # --------------------------------------------------------- + # PASS 3: The Unlearn (Mistake Recovery) + # --------------------------------------------------------- + print("\\n--- PASS 3: Unlearning and verifying re-query ---") + + # We simulate that the bot decided this classification was wrong and unlearns it + sae.unlearn_current_state(chaos_xml) + + # Give Qdrant a split second to process the deletion + time.sleep(0.5) + + start_time = time.time() + result_pass3 = sae.perceive(chaos_xml) + duration = time.time() - start_time + + print(f"Pass 3 completed in {duration:.2f}s. Result: {result_pass3}") + + # Assertions + assert spy_llm.call_count == 1, "LLM was NOT called after unlearning! Qdrant deletion failed." + assert result_pass3 == result_pass1, "LLM returned different result on third pass." + + print("\\n✅ Real LLM + Qdrant Learning/Unlearning cycle successfully validated!") diff --git a/tests/e2e/test_e2e_sae.py b/tests/e2e/test_e2e_sae.py index f29d2dd..a7cb285 100644 --- a/tests/e2e/test_e2e_sae.py +++ b/tests/e2e/test_e2e_sae.py @@ -4,89 +4,108 @@ Tests autonomous recovery from foreign apps, unknown modals, and learning. Uses REAL XML dumps from production sessions. """ -import pytest import os from unittest.mock import MagicMock, patch -from GramAddict.core.situational_awareness import ( - SituationalAwarenessEngine, SituationType, EscapeAction, SituationEpisodeDB -) -from GramAddict.core.device_facade import DeviceFacade +import pytest + +from GramAddict.core.device_facade import DeviceFacade +from GramAddict.core.situational_awareness import ( + EscapeAction, + SituationalAwarenessEngine, + SituationType, +) # ───────────────────────────────────────────────────── # Test Fixtures: Real-world XML scenarios # ───────────────────────────────────────────────────── + @pytest.fixture(autouse=True) def mock_screen_memory(): - with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None), \ - patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"): + with ( + patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None), + patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"), + ): yield + @pytest.fixture(autouse=True) def mock_telepathic_classifier(): with patch("GramAddict.core.llm_provider.query_telepathic_llm") as mock_llm: + def side_effect(model, url, system_prompt, user_prompt, use_local_edge): if "keyguard_status_view" in user_prompt or "lock_icon" in user_prompt: return '{"situation": "OBSTACLE_LOCKED_SCREEN"}' elif "permissioncontroller" in user_prompt: return '{"situation": "OBSTACLE_SYSTEM"}' - + # If it's a passive scaffold but no active modal markers, it's NORMAL - is_passive_only = "bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt - - if "survey_overlay_container" in user_prompt or "mystery_interstitial_container" in user_prompt or ("bottom_sheet_container" in user_prompt and not is_passive_only): + is_passive_only = ( + "bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt + ) + + if ( + "survey_overlay_container" in user_prompt + or "mystery_interstitial_container" in user_prompt + or ("bottom_sheet_container" in user_prompt and not is_passive_only) + ): return '{"situation": "OBSTACLE_MODAL"}' elif "feed_tab" in user_prompt: return '{"situation": "NORMAL"}' else: return '{"situation": "OBSTACLE_FOREIGN_APP"}' + mock_llm.side_effect = side_effect yield mock_llm + @pytest.fixture(autouse=True) def mock_fallback_llm(): with patch("GramAddict.core.llm_provider.query_llm") as mock_llm: + def side_effect(*args, **kwargs): - prompt = kwargs.get('prompt', args[2] if len(args) > 2 else "") + prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "") prompt_lower = prompt.lower() - + if "obstacle_foreign_app" in prompt_lower: return {"response": '{"action": "kill_foreign_apps", "x": 0, "y": 0, "reason": "Killing foreign app"}'} elif "obstacle_locked_screen" in prompt_lower: return {"response": '{"action": "unlock", "x": 0, "y": 0, "reason": "Unlocking device"}'} elif "close_friends" in prompt_lower: return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Safe fallback for follow sheet"}'} - + # Simulate LLM preferring BACK first for modals/dialogs if "back:0,0" not in prompt_lower: return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Trying safe BACK first"}'} - + if "not now" in prompt_lower or "später" in prompt_lower or "deny" in prompt_lower: return {"response": '{"action": "click", "x": 320, "y": 1850, "reason": "Found dismiss button"}'} - + return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback to back"}'} + mock_llm.side_effect = side_effect yield mock_llm -GOOGLE_SEARCH_XML = ''' + +GOOGLE_SEARCH_XML = """ -''' +""" -INSTAGRAM_HOME_XML = ''' +INSTAGRAM_HOME_XML = """ -''' +""" -INSTAGRAM_SURVEY_XML = ''' +INSTAGRAM_SURVEY_XML = """ @@ -96,9 +115,9 @@ INSTAGRAM_SURVEY_XML = ''' -''' +""" -UNKNOWN_MODAL_XML = ''' +UNKNOWN_MODAL_XML = """ @@ -108,9 +127,9 @@ UNKNOWN_MODAL_XML = ''' -''' +""" -PERMISSION_DIALOG_XML = ''' +PERMISSION_DIALOG_XML = """ @@ -119,9 +138,9 @@ PERMISSION_DIALOG_XML = ''' -''' +""" -LOCK_SCREEN_XML = ''' +LOCK_SCREEN_XML = """ @@ -129,13 +148,14 @@ LOCK_SCREEN_XML = ''' -''' +""" # ───────────────────────────────────────────────────── # Helpers # ───────────────────────────────────────────────────── + def make_mock_device(app_id="com.instagram.android"): device = MagicMock(spec=DeviceFacade) device.app_id = app_id @@ -154,6 +174,7 @@ def make_mock_device(app_id="com.instagram.android"): # PERCEPTION TESTS # ───────────────────────────────────────────────────── + class TestSAEPerception: """Tests that the SAE correctly classifies screen situations.""" @@ -171,6 +192,7 @@ class TestSAEPerception: def test_perceive_notification_shade(self): import os + dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml") try: with open(dump_path, "r") as f: @@ -180,7 +202,7 @@ class TestSAEPerception: result = sae.perceive(shade_xml) assert result == SituationType.OBSTACLE_FOREIGN_APP except FileNotFoundError: - pass # allow test format to compile if fixture accidentally not available + pass # allow test format to compile if fixture accidentally not available def test_perceive_system_permission_dialog(self): device = make_mock_device() @@ -201,10 +223,52 @@ class TestSAEPerception: result = sae.perceive(UNKNOWN_MODAL_XML) assert result == SituationType.OBSTACLE_MODAL + def test_perceive_randomized_chaos_modal(self, mock_telepathic_classifier): + """Generates completely random XML. Proves SAE passes dynamic state to VLM without hardcoded heuristics.""" + import uuid + + random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}" + random_text = f"Nonsense_Text_{uuid.uuid4().hex[:8]}" + random_button_text = f"Dismiss_{uuid.uuid4().hex[:8]}" + + chaos_xml = f""" + + + + + + + +""" + + device = make_mock_device() + sae = SituationalAwarenessEngine(device) + + # Override the mock behavior locally for this test to return OBSTACLE_MODAL + def local_side_effect(model, url, system_prompt, user_prompt, use_local_edge): + if random_text in user_prompt: + return '{"situation": "OBSTACLE_MODAL"}' + return '{"situation": "NORMAL"}' + + mock_telepathic_classifier.side_effect = local_side_effect + + result = sae.perceive(chaos_xml) + assert result == SituationType.OBSTACLE_MODAL + + # PROOF: The VLM was actually called, and the prompt contained our randomized strings! + mock_telepathic_classifier.assert_called_once() + _, kwargs = mock_telepathic_classifier.call_args + user_prompt = kwargs.get("user_prompt", "") + + id_suffix = random_id.split("/")[-1] + assert id_suffix in user_prompt, "Bot did not pass the random ID to VLM!" + assert random_text in user_prompt, "Bot did not pass the random text to VLM!" + assert random_button_text in user_prompt, "Bot did not pass the random button text to VLM!" + def test_perceive_action_blocked(self): blocked_xml = INSTAGRAM_HOME_XML.replace( 'text="" resource-id="com.instagram.android:id/feed_tab"', - 'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"' + 'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"', ) device = make_mock_device() sae = SituationalAwarenessEngine(device) @@ -227,13 +291,13 @@ class TestSAEPerception: """Passive scaffold containers (bottom_sheet_container_view, bottom_sheet_camera_container) must NOT be OBSTACLE_MODAL.""" device = make_mock_device() sae = SituationalAwarenessEngine(device) - + # XML containing navigation tabs + the passive scaffold container passive_xml = INSTAGRAM_HOME_XML.replace( '\n' '\n' - ' + + + + + + + +""" + device = make_mock_device() device.dump_hierarchy.side_effect = [ - UNKNOWN_MODAL_XML, # perceive: modal - UNKNOWN_MODAL_XML, # verify after BACK (failed) - UNKNOWN_MODAL_XML, # perceive again + chaos_xml, # perceive: modal + chaos_xml, # verify after BACK (failed) + chaos_xml, # perceive again + INSTAGRAM_HOME_XML, # verify after clicking randomized coords + ] + + # VLM Classifier override + def local_classifier(model, url, system_prompt, user_prompt, use_local_edge): + if random_text in user_prompt: + return '{"situation": "OBSTACLE_MODAL"}' + return '{"situation": "NORMAL"}' + + mock_telepathic_classifier.side_effect = local_classifier + + # VLM Fallback override (Action Solver) + def local_solver(*args, **kwargs): + prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "") + # Simulate real LLM: First it tries back, if 'back' is not in prompt + if "back:0,0" not in prompt.lower(): + return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Try back first"}'} + # Next time it sees the prompt, it finds the random button + if random_button_text in prompt: + # The bounds of our random button are [321,2001][541,2101] -> center is 431, 2051 + return {"response": '{"action": "click", "x": 431, "y": 2051, "reason": "Found chaos button"}'} + return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback"}'} + + mock_fallback_llm.side_effect = local_solver + + sae = SituationalAwarenessEngine(device) + with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"): + result = sae.ensure_clear_screen(max_attempts=5) + + assert result is True + # Proof that BACK was tried first + device.press.assert_called_with("back") + # Proof that the random coordinates were extracted and clicked + device.click.assert_called_once() + click_args = device.click.call_args + assert click_args[0] == ( + 431, + 2051, + ), f"Expected bot to click chaotic coordinates (431, 2051), but got {click_args[0]}" + + def test_recovers_from_unknown_modal_german(self): + device = make_mock_device() + device.dump_hierarchy.side_effect = [ + UNKNOWN_MODAL_XML, # perceive: modal + UNKNOWN_MODAL_XML, # verify after BACK (failed) + UNKNOWN_MODAL_XML, # perceive again INSTAGRAM_HOME_XML, # verify after clicking 'Später' ] sae = SituationalAwarenessEngine(device) - with patch.object(sae.episodes, 'recall', return_value=None), \ - patch.object(sae.episodes, 'learn'): + with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"): result = sae.ensure_clear_screen(max_attempts=5) assert result is True device.click.assert_called_once() @@ -409,7 +530,7 @@ class TestSAEAutonomousRecovery: def test_never_clicks_close_friends_on_follow_sheet(self): """CRITICAL REAL-WORLD BUG: Follow sheet has 'close_friends' row. SAE must NEVER click it — it adds the user to Close Friends!""" - follow_sheet_xml = ''' + follow_sheet_xml = """ @@ -418,16 +539,15 @@ class TestSAEAutonomousRecovery: - ''' + """ device = make_mock_device() device.dump_hierarchy.side_effect = [ - follow_sheet_xml, # perceive: modal + follow_sheet_xml, # perceive: modal INSTAGRAM_HOME_XML, # verify after BACK (worked!) ] sae = SituationalAwarenessEngine(device) - with patch.object(sae.episodes, 'recall', return_value=None), \ - patch.object(sae.episodes, 'learn'): + with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"): result = sae.ensure_clear_screen(max_attempts=5) assert result is True # CRITICAL: Must use BACK, never click any follow sheet button @@ -449,12 +569,12 @@ class TestSAEAutonomousRecovery: GOOGLE_SEARCH_XML, # attempt 5: perceive GOOGLE_SEARCH_XML, # attempt 5: verify (LLM failed) GOOGLE_SEARCH_XML, # attempt 6: perceive (escalate to app_start) - INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!) + INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!) ] sae = SituationalAwarenessEngine(device) # Mock LLM to return back action (simulating LLM also failing) - with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("back", reason="LLM says back")): + with patch.object(sae, "_plan_escape_via_llm", return_value=EscapeAction("back", reason="LLM says back")): result = sae.ensure_clear_screen(max_attempts=7) assert result is True device.app_start.assert_called() @@ -474,9 +594,10 @@ class TestSAEAutonomousRecovery: def test_action_blocked_raises_exception(self): """If Instagram blocks us, SAE must HALT — never try to dismiss.""" from GramAddict.core.exceptions import ActionBlockedError + blocked_xml = INSTAGRAM_HOME_XML.replace( 'text="" resource-id="com.instagram.android:id/feed_tab"', - 'text="Try again later" resource-id="com.instagram.android:id/dialog_container"' + 'text="Try again later" resource-id="com.instagram.android:id/dialog_container"', ) device = make_mock_device() device.dump_hierarchy.return_value = blocked_xml @@ -490,6 +611,7 @@ class TestSAEAutonomousRecovery: # LEARNING TESTS # ───────────────────────────────────────────────────── + class TestSAELearning: """Tests that SAE learns from experience and never repeats failures.""" @@ -536,15 +658,17 @@ class TestSAELearning: """When LLM returns 'false_positive', SAE must overwrite Qdrant and return True.""" device = make_mock_device() sae = SituationalAwarenessEngine(device) - + device.dump_hierarchy.return_value = INSTAGRAM_HOME_XML - + # Force the situation to be perceived as an OBSTACLE_MODAL initially - with patch.object(sae, 'perceive', return_value=SituationType.OBSTACLE_MODAL): + with patch.object(sae, "perceive", return_value=SituationType.OBSTACLE_MODAL): # Mock LLM to return 'false_positive' - with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("false_positive", reason="No modal found")): + with patch.object( + sae, "_plan_escape_via_llm", return_value=EscapeAction("false_positive", reason="No modal found") + ): result = sae.ensure_clear_screen(max_attempts=1, initial_xml=INSTAGRAM_HOME_XML) - + assert result is True mock_store_screen.assert_called_once() args, kwargs = mock_store_screen.call_args diff --git a/tests/e2e/test_full_e2e_android_sim.py b/tests/e2e/test_full_e2e_android_sim.py new file mode 100644 index 0000000..93b0cac --- /dev/null +++ b/tests/e2e/test_full_e2e_android_sim.py @@ -0,0 +1,217 @@ +import xml.etree.ElementTree as ET +from unittest.mock import MagicMock, patch + +import pytest + +from GramAddict.core.device_facade import DeviceFacade +from GramAddict.core.goap import GoalExecutor +from GramAddict.core.qdrant_memory import QdrantBase +from GramAddict.core.telepathic_engine import TelepathicEngine + + +class AndroidEnvironmentSimulator(DeviceFacade): + def __init__(self, device_id="sim", app_id="com.instagram.android", args=None): + self.device_id = device_id + self.app_id = app_id + self.args = args + self.deviceV2 = MagicMock() + self.deviceV2.info = {"displayWidth": 1080, "displayHeight": 2400, "screenOn": True} + + self.state_stack = ["home_feed"] + self.state_files = { + "home_feed": "tests/fixtures/home_feed_with_ad.xml", + "explore_grid": "tests/fixtures/explore_feed_dump.xml", + "post_detail": "tests/fixtures/organic_post.xml", + "user_profile": "tests/fixtures/user_profile_dump.xml", + } + + def _current_state(self): + return self.state_stack[-1] + + def dump_hierarchy(self): + current = self._current_state() + filepath = self.state_files[current] + with open(filepath, "r", encoding="utf-8") as f: + data = f.read() + print(f"📱 [Simulator] dump_hierarchy returning state: {current} (length: {len(data)})") + return data + + def _parse_bounds(self, bounds_str): + import re + + match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str) + if match: + return [int(x) for x in match.groups()] + return None + + def human_click(self, x, y): + # Simulate click translation to next state + xml_data = self.dump_hierarchy() + root = ET.fromstring(xml_data) + + clicked_nodes = [] + for node in root.iter("node"): + bounds_str = node.attrib.get("bounds", "") + bounds = self._parse_bounds(bounds_str) + if bounds: + x1, y1, x2, y2 = bounds + if x1 <= x <= x2 and y1 <= y <= y2: + area = (x2 - x1) * (y2 - y1) + clicked_nodes.append((area, node)) + + if not clicked_nodes: + return + + clicked_nodes.sort(key=lambda item: item[0]) + + for _, target in clicked_nodes: + content_desc = target.attrib.get("content-desc", "") or "" + res_id = target.attrib.get("resource-id", "") or "" + text = target.attrib.get("text", "") or "" + + current = self._current_state() + if current == "home_feed": + if "Search and explore" in content_desc or "search_tab" in res_id: + print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: home_feed -> explore_grid") + self.state_stack.append("explore_grid") + return + elif current == "explore_grid": + # In explore, anything the VLM clicks that has an image or button is likely a post + if "image_button" in res_id or "container" in res_id or target.attrib.get("clickable") == "true": + print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: explore_grid -> post_detail") + self.state_stack.append("post_detail") + return + elif current == "post_detail": + # Allow clicking either the post author or the comment author (both go to user_profile) + if 100 < x < 800 and 300 < y < 900: + print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: post_detail -> user_profile") + self.state_stack.append("user_profile") + return + + # If we get here, no transition happened + for _, target in clicked_nodes: + print( + f"📱 [Simulator] Click ({x}, {y}) fell through on: {target.attrib.get('resource-id')} / text={target.attrib.get('text')}" + ) + if not clicked_nodes: + print(f"📱 [Simulator] Click ({x}, {y}) fell outside ALL elements!") + + def click(self, x=None, y=None, obj=None): + if x is not None and y is not None: + self.human_click(x, y) + elif obj and isinstance(obj, dict) and "x" in obj: + self.human_click(obj["x"], obj["y"]) + + def press(self, key): + if key == "back": + if len(self.state_stack) > 1: + old_state = self.state_stack.pop() + print(f"📱 [Simulator] Back pressed. State Transition: {old_state} -> {self._current_state()}") + else: + print("📱 [Simulator] Back pressed at root state.") + + def _get_current_app(self): + return self.app_id + + def get_info(self): + return self.deviceV2.info + + def wake_up(self): + pass + + def unlock(self): + pass + + def shell(self, cmd): + return "" + + def swipe(self, sx, sy, ex, ey, duration=None): + print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})") + + def human_swipe(self, sx, sy, ex, ey, duration=None): + print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})") + + @property + def info(self): + return self.deviceV2.info + + +@pytest.fixture(autouse=True) +def setup_qdrant_isolation(): + """Prefix all Qdrant collections with test_sim_ so we don't pollute live data.""" + original_init = QdrantBase.__init__ + + def mocked_init(self, collection_name, *args, **kwargs): + test_collection = f"test_sim_{collection_name}" + original_init(self, test_collection, *args, **kwargs) + + with patch.object(QdrantBase, "__init__", new=mocked_init): + # We aggressively wipe these collections before running the test! + from GramAddict.core.qdrant_memory import NavigationMemoryDB + + qb = NavigationMemoryDB() + try: + qb.wipe_collection() + except: + pass + yield + + +def test_full_autonomous_sim_loop(monkeypatch): + """ + This test runs the real GoalExecutor with the real TelepathicEngine (VLM) + and real Qdrant (sandboxed via prefix) against a simulated Android environment. + """ + import urllib.request + + try: + urllib.request.urlopen("http://localhost:11434/", timeout=2) + except Exception: + pytest.skip("Ollama is not running. Live E2E sim requires LLM backend.") + + # 1. Create Simulator + sim_device = AndroidEnvironmentSimulator() + + # 2. Patch TelepathicEngine to NOT be mocked by conftest + engine = TelepathicEngine() + monkeypatch.setattr(TelepathicEngine, "get_instance", lambda: engine) + + # 3. Create context and GoalExecutor + from GramAddict.core.config import Config + + if not hasattr(Config(), "args"): + Config().args = MagicMock() + Config().args.use_nav_memory = True + Config().args.use_semantic_memory = True + + executor = GoalExecutor(sim_device, bot_username="testbot") + + # 4. Start an autonomous loop: We want to reach an organic post from the home feed + assert sim_device._current_state() == "home_feed" + + success = executor.achieve("open post", max_steps=10) + assert success is True + + # The VLM should have figured out: + # 1. Tap explore tab -> switches to "explore_grid" + # 2. Tap grid item -> switches to "post_detail" + assert sim_device._current_state() == "post_detail" + + # 5. Let's do another intent: view the user profile + success = executor.achieve("open post author profile", max_steps=5) + assert success is True + assert sim_device._current_state() == "user_profile" + + # 6. Now go back to the post + success = executor.achieve("open post", max_steps=5) + assert success is True + assert sim_device._current_state() == "post_detail" + + # 7. Check Qdrant Memory is actually populated + # We should have stored the state transitions in the goap_paths collection + from GramAddict.core.goap import PathMemory + + nav_db = PathMemory("testbot") + # verify at least some nodes exist + count = nav_db._db.client.count(nav_db._db.collection_name).count + assert count > 0, "Qdrant memory should have learned the paths!" diff --git a/tests/integration/test_bot_flow_interaction.py b/tests/integration/test_bot_flow_interaction.py index 85104d8..fa49fc9 100644 --- a/tests/integration/test_bot_flow_interaction.py +++ b/tests/integration/test_bot_flow_interaction.py @@ -1,14 +1,15 @@ -import pytest from unittest.mock import MagicMock, patch +import pytest + from GramAddict.core.bot_flow import ( + _align_active_post, + _extract_post_content, _run_zero_latency_feed_loop, _run_zero_latency_stories_loop, - _extract_post_content, is_ad, - _align_active_post ) -from GramAddict.core.session_state import SessionState + @pytest.fixture def mock_device(): @@ -20,12 +21,12 @@ def mock_device(): return device -@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance') +@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance") def test_extract_post_content(mock_get_telepathic): mock_engine = MagicMock() mock_engine.find_best_node.side_effect = [ {"original_attribs": {"text": "test_user"}}, - {"original_attribs": {"desc": "test description of image with more than 10 chars"}} + {"original_attribs": {"desc": "test description of image with more than 10 chars"}}, ] mock_get_telepathic.return_value = mock_engine xml = "" @@ -33,19 +34,21 @@ def test_extract_post_content(mock_get_telepathic): assert res["username"] == "test_user" assert "test description" in res["description"] -@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance') + +@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance") def test_extract_post_content_fallback_caption(mock_get_telepathic): mock_engine = MagicMock() mock_engine.find_best_node.side_effect = [{"original_attribs": {"text": "other_user"}}, None] mock_get_telepathic.return_value = mock_engine - xml = ''' + xml = """ - ''' + """ res = _extract_post_content(xml) assert res["username"] == "other_user" assert "this is a very long caption" in res["caption"] + def testis_ad(): assert is_ad('') == True assert is_ad('') == True @@ -53,7 +56,8 @@ def testis_ad(): assert is_ad('') == False assert is_ad('') == False -@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance') + +@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance") def test_align_active_post(mock_get_telepathic, mock_device): # Test snapping when post is far from ideal coordinates mock_engine = MagicMock() @@ -64,127 +68,176 @@ def test_align_active_post(mock_get_telepathic, mock_device): # The header is at 850px. Target is 250px. Diff is 600px. It should swipe. assert mock_device.swipe.called + def test_feed_loop_boredom_change_feed(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False mock_cognitive_stack["growth_brain"].evaluate_governance.return_value = "SHIFT_CONTEXT" - + configs = MagicMock() session_state = MagicMock() session_state.check_limit.return_value = (False, False, False, False) - - res = _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) + + res = _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) assert res == "BOREDOM_CHANGE_FEED" + def test_feed_loop_context_lost(mock_device, mock_cognitive_stack): # Simulate not having any feed markers 3 times mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, False, False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False - - mock_device.dump_hierarchy.return_value = "" # Blind - + + mock_device.dump_hierarchy.return_value = "" # Blind + # Needs telepathic engine mock - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, patch('GramAddict.core.bot_flow.dump_ui_state'): + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow.dump_ui_state"), + ): mock_instance = MockTelepathic.get_instance.return_value - mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}] # Pretend we have nodes so it doesn't trigger zero-node immediately - + mock_instance._extract_semantic_nodes.return_value = [ + {"x": 1, "y": 2} + ] # Pretend we have nodes so it doesn't trigger zero-node immediately + configs = MagicMock() session_state = MagicMock() session_state.check_limit.return_value = (False, False, False, False) - - res = _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) + + res = _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) assert res == "CONTEXT_LOST" + def test_feed_loop_zero_nodes(mock_device, mock_cognitive_stack): # Tests the Zero-Node recovery anomaly handler mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False - - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll: + + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, + ): mock_instance = MockTelepathic.get_instance.return_value - mock_instance._extract_semantic_nodes.return_value = [] # Zero interactive nodes - + mock_instance._extract_semantic_nodes.return_value = [] # Zero interactive nodes + configs = MagicMock() session_state = MagicMock() session_state.check_limit.return_value = (False, False, False, False) - - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) - + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + assert mock_device.press.called_with("back") assert mock_scroll.called + def test_feed_loop_ad_skip(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False - - mock_device.dump_hierarchy.return_value = ''' + + mock_device.dump_hierarchy.return_value = """ - ''' - - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \ - patch('GramAddict.core.bot_flow._align_active_post') as mock_align: + """ + + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, + patch("GramAddict.core.bot_flow._align_active_post") as mock_align, + ): mock_instance = MockTelepathic.get_instance.return_value mock_instance._extract_semantic_nodes.return_value = [{"x": 1}] mock_align.return_value = False - + configs = MagicMock() session_state = MagicMock() session_state.check_limit.return_value = (False, False, False, False) - - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) assert mock_scroll.called + def test_stories_loop_success(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False - + configs = MagicMock() configs.args.stories = "1" session_state = MagicMock() - - mock_device.dump_hierarchy.return_value = ''' + + mock_device.dump_hierarchy.return_value = """ - ''' - - with patch('GramAddict.core.bot_flow._humanized_click') as mock_click, patch('GramAddict.core.bot_flow.sleep'): + """ + + with patch("GramAddict.core.bot_flow._humanized_click") as mock_click, patch("GramAddict.core.bot_flow.sleep"): res = _run_zero_latency_stories_loop(mock_device, configs, session_state, mock_cognitive_stack) assert res == "FEED_EXHAUSTED" assert mock_click.called + def test_stories_loop_boredom(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = True - + configs = MagicMock() session_state = MagicMock() - - with patch('GramAddict.core.bot_flow.sleep'): + + with patch("GramAddict.core.bot_flow.sleep"): res = _run_zero_latency_stories_loop(mock_device, configs, session_state, mock_cognitive_stack) assert res == "BOREDOM_CHANGE_FEED" assert mock_device.press.called_with("back") + def test_start_bot_interrupt(): from GramAddict.core.bot_flow import start_bot - - # Mock all the heavy initialization - with patch('GramAddict.core.bot_flow.Config') as MockConfig, \ - patch('GramAddict.core.bot_flow.configure_logger'), \ - patch('GramAddict.core.bot_flow.check_if_updated'), \ - patch('GramAddict.core.benchmark_guard.check_model_benchmarks'), \ - patch('GramAddict.core.llm_provider.log_openrouter_burn'), \ - patch('GramAddict.core.bot_flow.create_device') as mock_create_device, \ - patch('GramAddict.core.bot_flow.set_time_delta') as mock_time_delta, \ - patch('GramAddict.core.bot_flow.SessionState') as MockSession, \ - patch('GramAddict.core.bot_flow.open_instagram', side_effect=KeyboardInterrupt()): + # Mock all the heavy initialization + with ( + patch("GramAddict.core.bot_flow.Config") as MockConfig, + patch("GramAddict.core.bot_flow.configure_logger"), + patch("GramAddict.core.bot_flow.check_if_updated"), + patch("GramAddict.core.benchmark_guard.check_model_benchmarks"), + patch("GramAddict.core.llm_provider.log_openrouter_burn"), + patch("GramAddict.core.bot_flow.create_device") as mock_create_device, + patch("GramAddict.core.bot_flow.set_time_delta") as mock_time_delta, + patch("GramAddict.core.bot_flow.SessionState") as MockSession, + patch("GramAddict.core.bot_flow.open_instagram", side_effect=KeyboardInterrupt()), + ): MockConfig.return_value.args.feed = True MockConfig.return_value.args.explore = False MockConfig.return_value.args.reels = False @@ -197,19 +250,20 @@ def test_start_bot_interrupt(): MockConfig.return_value.args.ai_embedding_url = "http://localhost:11434/api/chat" MockConfig.return_value.args.ai_embedding_model = "llama3" MockConfig.return_value.args.agent_strategy = "conservative" - + MockSession.inside_working_hours.return_value = (True, 0) - + with pytest.raises(KeyboardInterrupt): start_bot(username="test_user", device_id="123") + def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack): # This test hits the core interaction (Lines 900 - 1300) mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.85 - + configs = MagicMock() configs.args.likes_percentage = 100 configs.args.follow_percentage = 100 @@ -218,13 +272,15 @@ def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack): configs.args.ai_condenser_model = "test-model" configs.args.ai_condenser_url = "test-url" configs.args.dry_run_comments = False - + session_state = MagicMock() # If checking ALL, return a tuple of Falses. If specific limit like LIKES/COMMENTS, return False bool. - session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False - + session_state.check_limit.side_effect = ( + lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False + ) + # Needs to report a structure that has NO ad, HAS content, and HAS feed markers. - mock_device.dump_hierarchy.return_value = ''' + mock_device.dump_hierarchy.return_value = """ @@ -234,142 +290,182 @@ def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack): - ''' - + """ + # Ensure radome doesn't destroy our XML string mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x # Simulate that nav_graph transitions work mock_cognitive_stack["nav_graph"].do.return_value = True - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \ - patch('GramAddict.core.llm_provider.query_llm') as mock_llm, \ - patch('GramAddict.core.bot_flow.random.random', return_value=0.11), \ - patch('GramAddict.core.bot_flow.random.uniform', return_value=1.5), \ - patch('GramAddict.core.bot_flow.random.randint', return_value=1), \ - patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \ - patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \ - patch('GramAddict.core.bot_flow._humanized_click') as mock_click, \ - patch('GramAddict.core.stealth_typing.ghost_type') as mock_type: - + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract, + patch("GramAddict.core.llm_provider.query_llm") as mock_llm, + patch("GramAddict.core.bot_flow.random.random", return_value=0.11), + patch("GramAddict.core.bot_flow.random.uniform", return_value=1.5), + patch("GramAddict.core.bot_flow.random.randint", return_value=1), + patch("GramAddict.core.bot_flow._align_active_post", return_value=False), + patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, + patch("GramAddict.core.bot_flow._humanized_click") as mock_click, + patch("GramAddict.core.stealth_typing.ghost_type") as mock_type, + ): mock_extract.return_value = {"username": "legit_user", "description": "test image", "caption": ""} mock_instance = MockTelepathic.get_instance.return_value - mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2, "original_attribs": {"text": "This is a fantastic picture!"}}] + mock_instance._extract_semantic_nodes.return_value = [ + {"x": 1, "y": 2, "original_attribs": {"text": "This is a fantastic picture!"}} + ] mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False} mock_llm.return_value = {"response": "Great shot!"} - + mock_cognitive_stack["telepathic"] = mock_instance - + # We need to ensure that the configs allow interacting! configs.args.interact_percentage = 100 - - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) - + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + assert mock_click.called assert mock_type.called + def test_feed_loop_repost(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.85 - + configs = MagicMock() configs.args.repost_percentage = 100 configs.args.likes_percentage = 0 configs.args.comment_percentage = 0 - + session_state = MagicMock() - session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False - - mock_device.dump_hierarchy.return_value = ''' + session_state.check_limit.side_effect = ( + lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False + ) + + mock_device.dump_hierarchy.return_value = """ - ''' - + """ + mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x mock_cognitive_stack["nav_graph"].do.return_value = True - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow.random.random', return_value=0.11), \ - patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \ - patch('GramAddict.core.bot_flow._humanized_scroll'), \ - patch('GramAddict.core.bot_flow._humanized_click') as mock_click: - + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow.random.random", return_value=0.11), + patch("GramAddict.core.bot_flow._align_active_post", return_value=False), + patch("GramAddict.core.bot_flow._humanized_scroll"), + patch("GramAddict.core.bot_flow._humanized_click") as mock_click, + ): mock_instance = MockTelepathic.get_instance.return_value mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}] mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False} - + mock_cognitive_stack["telepathic"] = mock_instance configs.args.interact_percentage = 100 - + from GramAddict.core.bot_flow import _run_zero_latency_feed_loop - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + + def test_profile_learning_percentage_trigger(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False - mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 # Not high enough to trigger default - + mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 # Not high enough to trigger default + configs = MagicMock() - configs.args.profile_learning_percentage = 100 # Should force visit + configs.args.profile_learning_percentage = 100 # Should force visit configs.args.likes_percentage = 0 configs.args.comment_percentage = 0 - configs.args.follow_percentage = 0 # Won't trigger by follow chance either - + configs.args.follow_percentage = 0 # Won't trigger by follow chance either + session_state = MagicMock() - session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False - - mock_device.dump_hierarchy.return_value = ''' + session_state.check_limit.side_effect = ( + lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False + ) + + mock_device.dump_hierarchy.return_value = """ - ''' - + """ + mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x mock_cognitive_stack["nav_graph"].do.return_value = True - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \ - patch('GramAddict.core.bot_flow.random.random', return_value=0.5), \ - patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \ - patch('GramAddict.core.bot_flow._humanized_scroll'), \ - patch('GramAddict.core.bot_flow._interact_with_profile') as mock_interact: - + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract, + patch("GramAddict.core.bot_flow.random.random", return_value=0.5), + patch("GramAddict.core.bot_flow._align_active_post", return_value=False), + patch("GramAddict.core.bot_flow._humanized_scroll"), + patch("GramAddict.core.bot_flow._interact_with_profile") as mock_interact, + ): mock_extract.return_value = {"username": "legit_user", "description": "test image", "caption": ""} mock_instance = MockTelepathic.get_instance.return_value mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2, "original_attribs": {"text": "dummy"}}] mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False} - + mock_cognitive_stack["telepathic"] = mock_instance configs.args.interact_percentage = 100 - + from GramAddict.core.bot_flow import _run_zero_latency_feed_loop - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) - + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + assert mock_interact.called + def test_ai_learn_own_profile_triggers_goap(): - with patch('GramAddict.core.bot_flow.Config') as MockConfig, \ - patch('GramAddict.core.bot_flow.configure_logger'), \ - patch('GramAddict.core.bot_flow.check_if_updated'), \ - patch('GramAddict.core.benchmark_guard.check_model_benchmarks'), \ - patch('GramAddict.core.llm_provider.log_openrouter_burn'), \ - patch('GramAddict.core.llm_provider.prewarm_ollama_models'), \ - patch('GramAddict.core.bot_flow.create_device') as mock_create_device, \ - patch('GramAddict.core.bot_flow.set_time_delta'), \ - patch('GramAddict.core.bot_flow.SessionState') as MockSession, \ - patch('GramAddict.core.bot_flow.open_instagram', return_value=True), \ - patch('GramAddict.core.bot_flow.verify_and_switch_account', return_value=True), \ - patch('GramAddict.core.bot_flow.get_instagram_version', return_value="1.0"), \ - patch('GramAddict.core.goap.GoalExecutor') as MockGoalExecutor, \ - patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.llm_provider.query_llm') as mock_query, \ - patch('GramAddict.core.bot_flow.DojoEngine'), \ - patch('GramAddict.core.bot_flow.sleep'): - + with ( + patch("GramAddict.core.bot_flow.Config") as MockConfig, + patch("GramAddict.core.bot_flow.configure_logger"), + patch("GramAddict.core.bot_flow.check_if_updated"), + patch("GramAddict.core.benchmark_guard.check_model_benchmarks"), + patch("GramAddict.core.llm_provider.log_openrouter_burn"), + patch("GramAddict.core.llm_provider.prewarm_ollama_models"), + patch("GramAddict.core.bot_flow.create_device") as mock_create_device, + patch("GramAddict.core.bot_flow.set_time_delta"), + patch("GramAddict.core.bot_flow.SessionState") as MockSession, + patch("GramAddict.core.bot_flow.open_instagram", return_value=True), + patch("GramAddict.core.bot_flow.verify_and_switch_account", return_value=True), + patch("GramAddict.core.bot_flow.get_instagram_version", return_value="1.0"), + patch("GramAddict.core.goap.GoalExecutor") as MockGoalExecutor, + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.llm_provider.query_llm") as mock_query, + patch("GramAddict.core.bot_flow.DojoEngine"), + patch("GramAddict.core.bot_flow.sleep"), + ): MockConfig.return_value.args.ai_learn_own_profile = True MockConfig.return_value.args.agent_strategy = "aggressive_growth" MockConfig.return_value.args.capture_e2e_dumps = False @@ -379,26 +475,25 @@ def test_ai_learn_own_profile_triggers_goap(): MockConfig.return_value.args.stories = False MockConfig.return_value.args.working_hours = [10, 20] MockConfig.return_value.args.time_delta_session = 30 - + MockSession.inside_working_hours.return_value = (True, 0) - + mock_goap = MockGoalExecutor.get_instance.return_value mock_goap.achieve.return_value = True - - mock_telepathic = MockTelepathic.return_value # the class constructor is called inside start_bot - mock_telepathic._extract_semantic_nodes.return_value = [ - {"original_attribs": {"text": "my cool bio"}} - ] - + + mock_telepathic = MockTelepathic.return_value # the class constructor is called inside start_bot + mock_telepathic._extract_semantic_nodes.return_value = [{"original_attribs": {"text": "my cool bio"}}] + mock_query.return_value = {"persona": "cool dev", "vibe": "chill"} - + from GramAddict.core.bot_flow import start_bot + try: - with patch('GramAddict.core.bot_flow.random_sleep', side_effect=KeyboardInterrupt()): + with patch("GramAddict.core.bot_flow.random_sleep", side_effect=KeyboardInterrupt()): start_bot(username="testuser", device_id="123") except KeyboardInterrupt: pass - + mock_goap.achieve.assert_any_call("learn own profile") # resonance is created internally, so we can't easily assert on update_identity unless we patch ResonanceEngine too. # It's sufficient to know the GOAP goal was triggered. @@ -408,58 +503,75 @@ def test_profile_mismatch_recovery(mock_device, mock_cognitive_stack): mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True] mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False - mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 - + mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 + configs = MagicMock() - configs.args.profile_learning_percentage = 100 # Should force visit + configs.args.profile_learning_percentage = 100 # Should force visit configs.args.likes_percentage = 0 configs.args.comment_percentage = 0 - configs.args.follow_percentage = 0 - + configs.args.follow_percentage = 0 + session_state = MagicMock() - session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False - - feed_xml = ''' + session_state.check_limit.side_effect = ( + lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False + ) + + feed_xml = """ - ''' - profile_xml = ''' + """ + profile_xml = """ - ''' + """ call_count = [0] + def dump_hierarchy_mock(*args, **kwargs): call_count[0] += 1 return feed_xml if call_count[0] == 1 else profile_xml - + mock_device.dump_hierarchy.side_effect = dump_hierarchy_mock - + mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x mock_cognitive_stack["nav_graph"].do.return_value = True - with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \ - patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \ - patch('GramAddict.core.bot_flow.random.random', return_value=0.5), \ - patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \ - patch('GramAddict.core.bot_flow._humanized_scroll'), \ - patch('GramAddict.core.bot_flow._interact_with_profile') as mock_interact: - + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic, + patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract, + patch("GramAddict.core.bot_flow.random.random", return_value=0.5), + patch("GramAddict.core.bot_flow._align_active_post", return_value=False), + patch("GramAddict.core.bot_flow._humanized_scroll"), + patch("GramAddict.core.bot_flow._interact_with_profile") as mock_interact, + ): mock_extract.return_value = {"username": "amorextravel", "description": "test image", "caption": ""} mock_instance = MockTelepathic.get_instance.return_value mock_instance._extract_semantic_nodes.side_effect = [ - [{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 1st call at top of loop - [{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 2nd call before Targeted UX - [{"x": 1, "y": 2, "text": "ryanresatka", "resource_id": "com.instagram.android:id/action_bar_title"}] # 3rd call on profile + [{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 1st call at top of loop + [{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 2nd call before Targeted UX + [ + {"x": 1, "y": 2, "text": "ryanresatka", "resource_id": "com.instagram.android:id/action_bar_title"} + ], # 3rd call on profile ] mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False} - + mock_cognitive_stack["telepathic"] = mock_instance configs.args.interact_percentage = 100 - + from GramAddict.core.bot_flow import _run_zero_latency_feed_loop - _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack) - - assert mock_interact.call_args[0][2] == "ryanresatka", f"Expected ryanresatka but got {mock_interact.call_args[0][2]}" + + _run_zero_latency_feed_loop( + mock_device, + mock_cognitive_stack["zero_engine"], + mock_cognitive_stack["nav_graph"], + configs, + session_state, + "HomeFeed", + mock_cognitive_stack, + ) + + assert ( + mock_interact.call_args[0][2] == "ryanresatka" + ), f"Expected ryanresatka but got {mock_interact.call_args[0][2]}" diff --git a/tests/unit/perception/test_action_memory.py b/tests/unit/perception/test_action_memory.py new file mode 100644 index 0000000..df0e035 --- /dev/null +++ b/tests/unit/perception/test_action_memory.py @@ -0,0 +1,39 @@ +from unittest.mock import patch + +import pytest + +from GramAddict.core.perception.action_memory import ActionMemory +from GramAddict.core.perception.spatial_parser import SpatialNode + + +@pytest.fixture +def memory(): + with patch("GramAddict.core.qdrant_memory.UIMemoryDB") as MockDB: + mock_db = MockDB.return_value + yield ActionMemory(ui_memory=mock_db) + + +def test_track_click_stores_memory(memory): + node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node") + memory.track_click("tap test", node) + + assert memory._last_click_context is not None + assert memory._last_click_context["intent"] == "tap test" + + +def test_confirm_click_boosts_confidence(memory): + node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node") + memory.track_click("tap test", node) + memory.confirm_click() + + memory.ui_memory.boost_confidence.assert_called_once() + assert memory._last_click_context is None + + +def test_reject_click_decays_confidence(memory): + node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node") + memory.track_click("tap test", node) + memory.reject_click() + + memory.ui_memory.decay_confidence.assert_called_once() + assert memory._last_click_context is None diff --git a/tests/unit/perception/test_intent_resolver.py b/tests/unit/perception/test_intent_resolver.py new file mode 100644 index 0000000..af8ae8e --- /dev/null +++ b/tests/unit/perception/test_intent_resolver.py @@ -0,0 +1,37 @@ +from GramAddict.core.perception.intent_resolver import IntentResolver +from GramAddict.core.perception.spatial_parser import SpatialNode + + +def test_intent_resolver_finds_bottom_tab(): + resolver = IntentResolver() + + # A tab at the top + top_tab = SpatialNode(bounds=(0, 0, 100, 100), content_desc="Explore Tab", clickable=True) + # A tab at the bottom + bottom_tab = SpatialNode(bounds=(0, 2200, 100, 2300), content_desc="Explore Tab", clickable=True) + + # Intent resolver should prefer the one that geometrically matches the bottom navigation area + best_match = resolver.resolve("tap explore tab", [top_tab, bottom_tab]) + + assert best_match == bottom_tab + + +def test_intent_resolver_finds_button_by_text(): + resolver = IntentResolver() + + btn1 = SpatialNode(bounds=(0, 0, 100, 100), text="Follow", clickable=True) + btn2 = SpatialNode(bounds=(200, 200, 300, 300), text="Message", clickable=True) + + best_match = resolver.resolve("tap follow button", [btn1, btn2]) + + assert best_match == btn1 + + +def test_intent_resolver_returns_none_if_no_match(): + resolver = IntentResolver() + + btn = SpatialNode(bounds=(0, 0, 100, 100), text="Like", clickable=True) + + best_match = resolver.resolve("tap follow button", [btn]) + + assert best_match is None diff --git a/tests/unit/perception/test_spatial_parser.py b/tests/unit/perception/test_spatial_parser.py new file mode 100644 index 0000000..8b0d826 --- /dev/null +++ b/tests/unit/perception/test_spatial_parser.py @@ -0,0 +1,77 @@ +from GramAddict.core.perception.spatial_parser import SpatialNode, SpatialParser + +XML_FIXTURE = """ + + + + + + + + + + + + + +""" + + +class TestSpatialParser: + def test_parses_xml_into_spatial_nodes(self): + parser = SpatialParser() + root = parser.parse(XML_FIXTURE) + + assert root is not None + assert isinstance(root, SpatialNode) + assert root.bounds == (0, 0, 1080, 2400) + assert len(root.children) == 2 # The Bottom Nav and the Recycler View + + def test_extracts_all_clickable_nodes(self): + parser = SpatialParser() + root = parser.parse(XML_FIXTURE) + clickables = parser.get_clickable_nodes(root) + + assert len(clickables) == 3 + descriptions = [n.content_desc or n.text for n in clickables] + assert "Home Tab" in descriptions + assert "Explore Tab" in descriptions + assert "Like" in descriptions + + def test_spatial_containment(self): + parser = SpatialParser() + root = parser.parse(XML_FIXTURE) + + # Get the first post + post_nodes = [n for n in parser.get_all_nodes(root) if n.content_desc == "Post 1"] + assert len(post_nodes) == 1 + post = post_nodes[0] + + # Get the Like button + like_nodes = [n for n in parser.get_all_nodes(root) if n.text == "Like"] + assert len(like_nodes) == 1 + like_btn = like_nodes[0] + + # Spatial Containment: The Like button should be mathematically inside the Post + assert post.contains(like_btn) is True + + # The Like button should NOT contain the Post + assert like_btn.contains(post) is False + + # The Home Tab should NOT be in the Post + home_tabs = [n for n in parser.get_all_nodes(root) if n.content_desc == "Home Tab"] + assert post.contains(home_tabs[0]) is False + + def test_spatial_intersection(self): + parser = SpatialParser() + + # Node 1: Left side + n1 = SpatialNode(bounds=(0, 0, 100, 100)) + # Node 2: Overlapping Right side + n2 = SpatialNode(bounds=(50, 50, 150, 150)) + # Node 3: Far away + n3 = SpatialNode(bounds=(200, 200, 300, 300)) + + assert n1.intersects(n2) is True + assert n2.intersects(n1) is True + assert n1.intersects(n3) is False diff --git a/tests/unit/test_bot_flow_unlearn.py b/tests/unit/test_bot_flow_unlearn.py new file mode 100644 index 0000000..05203f3 --- /dev/null +++ b/tests/unit/test_bot_flow_unlearn.py @@ -0,0 +1,65 @@ +from unittest.mock import MagicMock, patch + +from GramAddict.core.bot_flow import _run_zero_latency_feed_loop + + +def test_bot_flow_unlearns_on_context_loss(): + """Prove that bot_flow calls unlearn_current_state when context is completely lost (3 misses).""" + device = MagicMock() + # Provide a dummy dump hierarchy + device.dump_hierarchy.return_value = "" + device.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400} + + session_state = MagicMock() + + # We will patch SituationalAwarenessEngine + with patch("GramAddict.core.situational_awareness.SituationalAwarenessEngine") as MockSAE: + # We need mock SAE to return OBSTACLE_MODAL to trigger the first condition + # Wait, the code has two paths: `has_obstacle` or `not has_feed_markers`. + # If we return `False` for has_feed_markers, it hits the second path. + + mock_sae_instance = MockSAE.return_value + # perceive needs to return something that is not OBSTACLE_MODAL so we hit the feed markers path + mock_sae_instance.perceive.return_value = "EXPLORE_GRID" + + # Act: _run_zero_latency_feed_loop runs a loop. + # Since has_feed_markers is always False, it will increment misses 3 times and return "CONTEXT_LOST". + # We also need to mock TelepathicEngine so it doesn't crash on misses == 2. + with ( + patch("GramAddict.core.bot_flow.TelepathicEngine") as MockTelepathic, + patch("GramAddict.core.bot_flow.dump_ui_state"), + ): + mock_telepathic_instance = MockTelepathic.get_instance.return_value + mock_telepathic_instance.find_best_node.return_value = None + mock_telepathic_instance._extract_semantic_nodes.return_value = [MagicMock()] + + mock_cognitive_stack = MagicMock() + dopamine_mock = MagicMock() + dopamine_mock.is_app_session_over.return_value = False + dopamine_mock.wants_to_doomscroll.return_value = False + + def stack_get(key): + if key == "radome": + return None + elif key == "dopamine": + return dopamine_mock + return MagicMock() + + mock_cognitive_stack.get.side_effect = stack_get + + with patch("GramAddict.core.bot_flow.is_ad", return_value=False): + result = _run_zero_latency_feed_loop( + device=device, + zero_engine=MagicMock(), + nav_graph=MagicMock(), + configs=MagicMock(), + session_state=session_state, + job_target="test_feed", + cognitive_stack=mock_cognitive_stack, + ) + + # Assert (RED) + assert result == "CONTEXT_LOST" + + # SAE should have been told to unlearn the current state because of context loss + mock_sae_instance.unlearn_current_state.assert_called_with("") diff --git a/tests/unit/test_explore_grid_navigation.py b/tests/unit/test_explore_grid_navigation.py index 501b36d..e6895a9 100644 --- a/tests/unit/test_explore_grid_navigation.py +++ b/tests/unit/test_explore_grid_navigation.py @@ -8,67 +8,91 @@ Reproduces the exact production failure from 2026-04-16 22:59 where the bot: These tests MUST fail before the fix and pass after. """ -import pytest -import re -from GramAddict.core.telepathic_engine import TelepathicEngine +import re + +from GramAddict.core.telepathic_engine import TelepathicEngine # ── Realistic node fixtures extracted from live XML dump 2026-04-16_22-59-53 ── + def make_node(x, y, bounds, semantic, res_id="com.instagram.android:id/dummy", text="", desc=""): - m = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds) + m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds) l, t, r, b = map(int, m.groups()) if m else (0, 0, 0, 0) return { - "x": x, "y": y, - "width": r - l, "height": b - t, "area": (r - l) * (b - t), + "x": x, + "y": y, + "width": r - l, + "height": b - t, + "area": (r - l) * (b - t), "raw_bounds": bounds, "semantic_string": semantic, "resource_id": res_id, "class_name": "android.widget.FrameLayout", "selected": False, - "original_attribs": {"text": text, "desc": desc} + "original_attribs": {"text": text, "desc": desc}, } EXPLORE_GRID_NODES = [ # Row 1, Col 1 — this is what "first image" should match - make_node(178, 559, "[0,321][356,797]", - "description: '4 photos by Patsy Weingart at row 1, column 1', id context: 'grid card layout container'", - res_id="com.instagram.android:id/grid_card_layout_container", - desc="4 photos by Patsy Weingart at row 1, column 1"), + make_node( + 178, + 559, + "[0,321][356,797]", + "description: '4 photos by Patsy Weingart at row 1, column 1', id context: 'grid card layout container'", + res_id="com.instagram.android:id/grid_card_layout_container", + desc="4 photos by Patsy Weingart at row 1, column 1", + ), # Row 1, Col 1 — child image_button (same area, no semantic info) - make_node(178, 558, "[0,321][356,796]", - "id context: 'image button'", - res_id="com.instagram.android:id/image_button"), + make_node( + 178, 558, "[0,321][356,796]", "id context: 'image button'", res_id="com.instagram.android:id/image_button" + ), # Row 1, Col 2 - make_node(540, 559, "[362,321][718,797]", - "description: 'Photo by Barbara at Row 1, Column 2', id context: 'grid card layout container'", - res_id="com.instagram.android:id/grid_card_layout_container", - desc="Photo by Barbara at Row 1, Column 2"), - make_node(540, 558, "[362,321][718,796]", - "id context: 'image button'", - res_id="com.instagram.android:id/image_button"), + make_node( + 540, + 559, + "[362,321][718,797]", + "description: 'Photo by Barbara at Row 1, Column 2', id context: 'grid card layout container'", + res_id="com.instagram.android:id/grid_card_layout_container", + desc="Photo by Barbara at Row 1, Column 2", + ), + make_node( + 540, 558, "[362,321][718,796]", "id context: 'image button'", res_id="com.instagram.android:id/image_button" + ), # Row 2, Col 2 - make_node(540, 1041, "[362,803][718,1279]", - "description: 'Photo by Garima Bhaskar at Row 2, Column 2', id context: 'grid card layout container'", - res_id="com.instagram.android:id/grid_card_layout_container", - desc="Photo by Garima Bhaskar at Row 2, Column 2"), - make_node(540, 1040, "[362,803][718,1278]", - "id context: 'image button'", - res_id="com.instagram.android:id/image_button"), + make_node( + 540, + 1041, + "[362,803][718,1279]", + "description: 'Photo by Garima Bhaskar at Row 2, Column 2', id context: 'grid card layout container'", + res_id="com.instagram.android:id/grid_card_layout_container", + desc="Photo by Garima Bhaskar at Row 2, Column 2", + ), + make_node( + 540, 1040, "[362,803][718,1278]", "id context: 'image button'", res_id="com.instagram.android:id/image_button" + ), # Row 3, Col 1 — this is what the VLM wrongly picked - make_node(178, 1523, "[0,1285][356,1761]", - "description: '4 photos by Soul Of Nature Photography at row 3, column 1', id context: 'grid card layout container'", - res_id="com.instagram.android:id/grid_card_layout_container", - desc="4 photos by Soul Of Nature Photography at row 3, column 1"), - make_node(178, 1522, "[0,1285][356,1760]", - "id context: 'image button'", - res_id="com.instagram.android:id/image_button"), + make_node( + 178, + 1523, + "[0,1285][356,1761]", + "description: '4 photos by Soul Of Nature Photography at row 3, column 1', id context: 'grid card layout container'", + res_id="com.instagram.android:id/grid_card_layout_container", + desc="4 photos by Soul Of Nature Photography at row 3, column 1", + ), + make_node( + 178, 1522, "[0,1285][356,1760]", "id context: 'image button'", res_id="com.instagram.android:id/image_button" + ), # Search bar - make_node(487, 219, "[32,173][943,265]", - "text: 'Search', id context: 'action bar search edit text'", - res_id="com.instagram.android:id/action_bar_search_edit_text", - text="Search"), + make_node( + 487, + 219, + "[32,173][943,265]", + "text: 'Search', id context: 'action bar search edit text'", + res_id="com.instagram.android:id/action_bar_search_edit_text", + text="Search", + ), ] @@ -87,17 +111,18 @@ class TestBlacklistPoisoning: engine = TelepathicEngine() # Clear any persisted state to test pure logic engine._blacklist = {} - + # Simulate the flow: the engine "clicked" an image_button and it failed TelepathicEngine._last_click_context = { "intent": "first image in explore grid", "semantic_string": "id context: 'image button'", - "x": 178, "y": 558, - "timestamp": 1000 + "x": 178, + "y": 558, + "timestamp": 1000, } - + engine.reject_click("first image in explore grid") - + # The blacklist should NOT contain this generic entry blacklisted = engine._blacklist.get("first image in explore grid", []) assert "id context: 'image button'" not in blacklisted, ( @@ -123,18 +148,20 @@ class TestExploreGridFastPath: # Build a minimal XML that the engine can parse — but we test the fast-path # directly by calling find_best_node with mocked extraction from unittest.mock import patch - - with patch.object(engine, '_extract_semantic_nodes', return_value=EXPLORE_GRID_NODES): - with patch.object(engine, '_is_instagram_context', return_value=True): - result = engine.find_best_node("", "first image in explore grid", min_confidence=0.82, device=None) - + + with patch.object(engine, "_extract_semantic_nodes", return_value=EXPLORE_GRID_NODES): + with patch.object(engine, "_is_instagram_context", return_value=True): + result = engine.find_best_node( + "", "first image in explore grid", min_confidence=0.82, device=None + ) + assert result is not None, ( "Grid Fast-Path returned None for 'first image in explore grid'. " "This forces every explore grid tap to use the expensive VLM fallback." ) - assert any(k in result.get("semantic", "").lower() for k in ["image button", "grid card layout container"]), ( - f"Grid Fast-Path selected wrong node type: {result.get('semantic')}" - ) + assert any( + k in result.get("semantic_string", "").lower() for k in ["image button", "grid card layout container"] + ), f"Grid Fast-Path selected wrong node type: {result.get('semantic_string')}" def test_grid_fastpath_prefers_topmost_row(self): """ @@ -142,13 +169,15 @@ class TestExploreGridFastPath: topmost one (smallest Y = row 1) since the intent says 'first'. """ engine = TelepathicEngine() - + from unittest.mock import patch - - with patch.object(engine, '_extract_semantic_nodes', return_value=EXPLORE_GRID_NODES): - with patch.object(engine, '_is_instagram_context', return_value=True): - result = engine.find_best_node("", "first image in explore grid", min_confidence=0.82, device=None) - + + with patch.object(engine, "_extract_semantic_nodes", return_value=EXPLORE_GRID_NODES): + with patch.object(engine, "_is_instagram_context", return_value=True): + result = engine.find_best_node( + "", "first image in explore grid", min_confidence=0.82, device=None + ) + if result is not None: # Row 1 items have y ≈ 559, Row 3 items have y ≈ 1523 assert result["y"] < 800, ( @@ -172,25 +201,26 @@ class TestVerifySuccessExploreGrid: (no feed markers), verify_success must return None (Inconclusive). """ engine = TelepathicEngine() - + # Simulate that we just clicked a grid item TelepathicEngine._last_click_context = { "intent": "first image in explore grid", "semantic_string": "description: '4 photos by Patsy Weingart at row 1, column 1'", - "x": 178, "y": 559, - "timestamp": 1000 + "x": 178, + "y": 559, + "timestamp": 1000, } - + # Post-click XML still shows the explore grid (no feed markers) still_on_grid_xml = """ - """ - + result = engine.verify_success("first image in explore grid", still_on_grid_xml) assert result is None, "verify_success should be inconclusive (None) when still on explore grid" @@ -200,14 +230,15 @@ class TestVerifySuccessExploreGrid: verify_success must return True. """ engine = TelepathicEngine() - + TelepathicEngine._last_click_context = { "intent": "first image in explore grid", "semantic_string": "description: '4 photos by Patsy Weingart at row 1, column 1'", - "x": 178, "y": 559, - "timestamp": 1000 + "x": 178, + "y": 559, + "timestamp": 1000, } - + # Post-click XML shows a feed post (has feed markers) post_view_xml = """ @@ -217,6 +248,6 @@ class TestVerifySuccessExploreGrid: """ - + result = engine.verify_success("first image in explore grid", post_view_xml) assert result is True, "verify_success should pass when post view is visible" diff --git a/tests/unit/test_goap_unlearn.py b/tests/unit/test_goap_unlearn.py new file mode 100644 index 0000000..4cf13b2 --- /dev/null +++ b/tests/unit/test_goap_unlearn.py @@ -0,0 +1,62 @@ +from unittest.mock import MagicMock, patch + +from GramAddict.core.goap import GoalExecutor +from GramAddict.core.screen_topology import ScreenType + + +def test_goap_unlearns_transition_on_back_press_trap(): + """Prove that GOAP forgets a specific topological transition when it gets trapped and spams 'back'.""" + device = MagicMock() + orchestrator = GoalExecutor(device, "testuser") + + # Mocking internal state + start_screen = ScreenType.HOME_FEED + goal = "open messages" + steps_taken = [ + {"action": "tap explore tab"}, + {"action": "press back"}, + {"action": "press back"}, + {"action": "press back"}, + ] + + def mock_exec(*args, **kwargs): + print("EXECUTING:", args, kwargs) + return True + + orchestrator._execute_action = MagicMock(side_effect=mock_exec) # "press back" succeeds + orchestrator.path_memory = MagicMock() + orchestrator.path_memory.recall_path.return_value = None + # To test the back-press circuit breaker, we just need to feed it 3 "press back" actions. + + # Since achieve is complex, let's just test that the required logic + # exists inside it. The circuit breaker is in the "EXECUTE" block of achieve. + # We will mock the planner to return "press back" 3 times. + orchestrator.planner.plan_next_step = MagicMock(side_effect=[["press back"], ["press back"], ["press back"], []]) + orchestrator.perceive = MagicMock( + return_value={"screen_type": ScreenType.EXPLORE_GRID, "available_actions": ["mock"]} + ) + + with patch("GramAddict.core.qdrant_memory.NavigationMemoryDB") as MockNavDB: + mock_nav_instance = MockNavDB.return_value + + # We need to simulate that `steps_taken` already had "tap explore tab" + # However, achieve starts with an empty `steps_taken`. + # So we mock the internal variables if possible, but they are local. + # Alternatively, we make the planner return "tap explore tab", then 3 "press back"s. + orchestrator.planner.plan_next_step = MagicMock( + side_effect=["tap explore tab", "press back", "press back", "press back", "press back", None] + ) + + # Act + result = orchestrator.achieve(goal, max_steps=10) + + # Assert (RED) + assert result is False + + # Did it forget the path? (learn_path with success=False) + assert orchestrator.path_memory.learn_path.called + + # Did it unlearn the transition? + print("MockNavDB calls:", MockNavDB.mock_calls) + print("NavInstance calls:", mock_nav_instance.mock_calls) + mock_nav_instance.unlearn_transition.assert_called_once_with(ScreenType.EXPLORE_GRID.value, "tap explore tab") diff --git a/tests/unit/test_profile_interaction_edges.py b/tests/unit/test_profile_interaction_edges.py index 258f4d3..bb951f7 100644 --- a/tests/unit/test_profile_interaction_edges.py +++ b/tests/unit/test_profile_interaction_edges.py @@ -1,12 +1,12 @@ -import pytest from unittest.mock import MagicMock -from GramAddict.core.telepathic_engine import TelepathicEngine + from GramAddict.core.q_nav_graph import QNavGraph +from GramAddict.core.telepathic_engine import TelepathicEngine class TestProfileInteractionSync: """ - TDD Tests: Reproduces the 'Already Followed -> Favorites' and 'No Story Ring' + TDD Tests: Reproduces the 'Already Followed -> Favorites' and 'No Story Ring' bugs from 2026-04-17 live run. """ @@ -24,68 +24,84 @@ class TestProfileInteractionSync: the engine must skip the click to prevent opening the Favorites/Mute bottom sheet. """ # Simulate a profile where the user is already followed - viable_nodes = [{ - "semantic_string": "id context: 'profile header user action follow button', text: 'Following'", - "x": 500, "y": 600, - "width": 100, "height": 50, - "area": 5000, - "class_name": "android.widget.Button", - "resource_id": "com.instagram.android:id/button", - "original_attribs": {"text": "Following"} - }] - + viable_nodes = [ + { + "semantic_string": "id context: 'profile header user action follow button', text: 'Following'", + "x": 500, + "y": 600, + "width": 100, + "height": 50, + "area": 5000, + "class_name": "android.widget.Button", + "resource_id": "com.instagram.android:id/button", + "original_attribs": {"text": "Following"}, + } + ] + # Test vector-based matching fallback self.engine._blacklist = {} - - # Mock the extraction to avoid needing valid complex XML - self.engine._extract_semantic_nodes = MagicMock(return_value=viable_nodes) + + # Mock the parsing instead of old extraction + self.engine._parser = MagicMock() + self.engine._parser.parse.return_value = MagicMock() + + # We must return SpatialNode objects for the new architecture + from GramAddict.core.perception.spatial_parser import SpatialNode + + mock_node = SpatialNode(MagicMock()) + mock_node.resource_id = "com.instagram.android:id/button" + mock_node.text = "Following" + mock_node.bounds = [500, 600, 600, 650] + self.engine._parser.get_clickable_nodes.return_value = [mock_node] + self.engine._structural_sanity_check = MagicMock(return_value=True) self.engine._is_instagram_context = MagicMock(return_value=True) - + + # Mock resolver to return our mock node + self.engine._resolver = MagicMock() + self.engine._resolver.resolve.return_value = mock_node + result = self.engine.find_best_node("", "tap follow button on profile", device=self.mock_device) - - # We must intercept it in TelepathicEngine before VLM is called - # Wait, find_best_node falls back to VLM if vector score is low. - # But if we inject it into memory, it triggers stage 1 - self.engine._memory = { - "tap follow button on profile": ["id context: 'profile header user action follow button', text: 'Following'"] - } - TelepathicEngine._instance = self.engine - - result = self.engine.find_best_node("", "tap follow button on profile", device=self.mock_device) - + assert result is not None, "Engine should return a skip result, not None" assert result.get("skip") is True, "Must return skip: True to prevent Favorites menu from opening" assert result.get("semantic") == "already_followed" - def test_story_ring_not_present_skips_click(self): """ - If no story ring is explicitly in the XML, bot_flow should not execute + If no story ring is explicitly in the XML, bot_flow should not execute the transition (simulated here by checking our XML evaluation logic). """ - xml_without_story = ''' + xml_without_story = """ - ''' - - has_story = "reel_ring" in xml_without_story or "unseen story" in xml_without_story.lower() or "story von" in xml_without_story.lower() - + """ + + has_story = ( + "reel_ring" in xml_without_story + or "unseen story" in xml_without_story.lower() + or "story von" in xml_without_story.lower() + ) + assert has_story is False, "Logic falsely identified a story when there is only a generic profile picture" def test_story_ring_present_allows_click(self): """ If a story ring is present, the logic should allow the interaction. """ - xml_with_story = ''' + xml_with_story = """ - ''' - - has_story = "reel_ring" in xml_with_story or "unseen story" in xml_with_story.lower() or "story von" in xml_with_story.lower() - + """ + + has_story = ( + "reel_ring" in xml_with_story + or "unseen story" in xml_with_story.lower() + or "story von" in xml_with_story.lower() + ) + assert has_story is True, "Logic failed to identify active story ring" diff --git a/tests/unit/test_sae_tesla_upgrade.py b/tests/unit/test_sae_tesla_upgrade.py new file mode 100644 index 0000000..a58ee04 --- /dev/null +++ b/tests/unit/test_sae_tesla_upgrade.py @@ -0,0 +1,79 @@ +from unittest.mock import MagicMock, patch + +from GramAddict.core.situational_awareness import EscapeAction, SituationalAwarenessEngine, SituationEpisodeDB + + +class TestSAETeslaUpgrade: + def setup_method(self): + self.device_mock = MagicMock() + self.sae = SituationalAwarenessEngine(self.device_mock) + + def test_sae_foreground_extraction(self): + """ + Ensures that modals/popups located at the END of the XML document + (highest Z-index) are prioritized in the compressed signature. + The old system truncated at elements[:50], missing the actual popup. + """ + # Create an XML with 60 background nodes, and 1 dialog node at the end + xml_dump = "\n\n" + for i in range(60): + xml_dump += f' \n' + + # The critical modal at the end + xml_dump += ' \n' + xml_dump += "" + + compressed = self.sae._compress_xml(xml_dump) + + # The compressed string MUST contain the dialog container + assert "dialog_container" in compressed, "Foreground extraction failed: modal was truncated!" + assert "Not Now" in compressed, "Foreground extraction failed: modal text was truncated!" + + # It should prioritize the END of the document, so feed_item_0 should ideally be gone if capped at 50 + assert "feed_item_0" not in compressed, "Background elements are still being prioritized over foreground!" + + def test_sae_structural_generalization(self): + """ + Ensures that dynamic user content is stripped to allow cross-post modal generalization, + while short, static UI text (like "OK", "Cancel") is preserved. + """ + xml_dump = """ + + + + + """ + + compressed = self.sae._compress_xml(xml_dump) + + # Long dynamic text should be stripped or truncated to not pollute the vector space + assert "This is a very long user comment" not in compressed, "Dynamic text > 20 chars was not stripped!" + assert "text=''" in compressed or "This is a very lo" not in compressed + # Short static text should be kept + assert "OK" in compressed, "Short static UI text was incorrectly stripped!" + + @patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new=True) + def test_sae_negative_reinforcement(self): + """ + Ensures that failed escapes decay the confidence of the vector, + and eventually purge it, instead of just storing a useless 0.0 vector alongside it. + """ + db = SituationEpisodeDB() + + # We need to mock db._db.client directly since it's an instance property + mock_client = MagicMock() + db._db._client = mock_client + db._db.client = mock_client + + with patch.object(db._db, "upsert_point") as mock_upsert: + # Mock retrieve to return an existing point with confidence 0.4 + mock_payload = {"confidence": 0.4, "action": {"action_type": "back", "x": 0, "y": 0, "reason": ""}} + mock_client.retrieve.return_value = [MagicMock(payload=mock_payload)] + + # Simulate a FAILURE + action = EscapeAction("back") + db.learn("some_signature", action, success=False) + + # Verify that it fetched the current confidence and updated it, or deleted it if < 0.1 + # If confidence was 0.4 and delta is -0.5, it drops to -0.1 -> DELETED + mock_client.delete.assert_called_once() diff --git a/tests/unit/test_sae_unlearn.py b/tests/unit/test_sae_unlearn.py new file mode 100644 index 0000000..fd5d9f8 --- /dev/null +++ b/tests/unit/test_sae_unlearn.py @@ -0,0 +1,22 @@ +from unittest.mock import MagicMock, patch + +from GramAddict.core.situational_awareness import SituationalAwarenessEngine + + +def test_sae_has_unlearn_current_state(): + """Prove that SituationalAwarenessEngine exposes unlearn_current_state to heal from poisoned context.""" + device = MagicMock() + sae = SituationalAwarenessEngine(device) + + # Mocking internal compression and the screen_memory dependency + sae._compress_xml = MagicMock(return_value="") + + with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB") as MockScreenMemoryDB: + mock_db_instance = MockScreenMemoryDB.return_value + + # If unlearn_current_state does not exist, AttributeError (RED) + sae.unlearn_current_state("") + + # Verify it compresses and delegates to purge_screen + sae._compress_xml.assert_called_once_with("") + mock_db_instance.purge_screen.assert_called_once_with("") diff --git a/tests/unit/test_self_healing_memory.py b/tests/unit/test_self_healing_memory.py new file mode 100644 index 0000000..2a1af60 --- /dev/null +++ b/tests/unit/test_self_healing_memory.py @@ -0,0 +1,50 @@ +from unittest.mock import MagicMock, PropertyMock, patch + +from GramAddict.core.qdrant_memory import NavigationMemoryDB, QdrantBase, ScreenMemoryDB + + +class DummyQdrantBase(QdrantBase): + def __init__(self): + self.client = MagicMock() + self.collection_name = "test_collection" + + +@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock) +def test_qdrant_base_has_delete_point(mock_is_connected): + """Prove that QdrantBase implements delete_point for autonomous unlearning.""" + mock_is_connected.return_value = True + db = DummyQdrantBase() + # If delete_point does not exist, this will raise AttributeError (RED) + db.generate_uuid = MagicMock(return_value="test-uuid-1234") + + result = db.delete_point("test-seed") + + # Assert + db.client.delete.assert_called_once_with(collection_name="test_collection", points_selector=["test-uuid-1234"]) + assert result is True + + +@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock) +def test_screen_memory_has_purge_screen(mock_is_connected): + """Prove that ScreenMemoryDB exposes purge_screen to heal poisoned classifications.""" + mock_is_connected.return_value = True + db = ScreenMemoryDB() + db.client = MagicMock() + db.delete_point = MagicMock() + + # If purge_screen does not exist, AttributeError (RED) + db.purge_screen("") + db.delete_point.assert_called_once_with("") + + +@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock) +def test_navigation_memory_has_unlearn_transition(mock_is_connected): + """Prove that NavigationMemoryDB exposes unlearn_transition to destroy trap paths.""" + mock_is_connected.return_value = True + db = NavigationMemoryDB() + db.client = MagicMock() + db.delete_point = MagicMock() + + # If unlearn_transition does not exist, AttributeError (RED) + db.unlearn_transition("HOME_FEED", "tap explore tab") + db.delete_point.assert_called_once_with("HOME_FEED_tap explore tab") diff --git a/tests/unit/test_telepathic_brevity_bonus.py b/tests/unit/test_telepathic_brevity_bonus.py deleted file mode 100644 index 2df89f0..0000000 --- a/tests/unit/test_telepathic_brevity_bonus.py +++ /dev/null @@ -1,38 +0,0 @@ -from GramAddict.core.telepathic_engine import TelepathicEngine - - -def test_brevity_bonus_prioritizes_short_labels(): - """ - Tests that the brevity bonus correctly prioritizes short, exact matches - over longer matches that contain the same keywords. - """ - engine = TelepathicEngine() - - # A short, precise button - short_node = { - "x": 100, - "y": 200, - "area": 500, - "semantic_string": "text: 'Profile', id context: 'tab bar profile'", - "resource_id": "tab_bar_profile", - "original_attribs": {"desc": "", "text": "Profile"}, - } - - # A long, descriptive text that happens to contain "Profile" - long_node = { - "x": 100, - "y": 300, - "area": 5000, - "semantic_string": "text: 'Visit my profile to see more photos', id context: 'feed post text'", - "resource_id": "feed_post_text", - "original_attribs": {"desc": "", "text": "Visit my profile to see more photos"}, - } - - nodes = [long_node, short_node] - - # "profile" is the intent - result = engine._keyword_match_score("profile", nodes) - - assert result is not None, "Failed to extract node via fast path" - # The short node should win because of the brevity bonus (0.2) - assert "tab bar profile" in result["semantic"], "Brevity bonus failed to prioritize the shorter label" diff --git a/tests/unit/test_telepathic_confidence.py b/tests/unit/test_telepathic_confidence.py deleted file mode 100644 index 79895a0..0000000 --- a/tests/unit/test_telepathic_confidence.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest -from GramAddict.core.telepathic_engine import TelepathicEngine - -def test_extract_post_author_confidence(): - """ - Tests that the TelepathicEngine can confidently extract the post author - header node from a standard feed XML dump, even if it falls back to the - fast path or embeddings. - """ - engine = TelepathicEngine() - - # A generic Feed post author node - author_node = { - "x": 100, "y": 200, "area": 500, - "semantic_string": "description: 'fiona.dawson', id context: 'row feed photo profile name'", - "resource_id": "row_feed_photo_profile_name", - "original_attribs": {"desc": "fiona.dawson", "text": "fiona.dawson"} - } - - # A generic Feed post image node - image_node = { - "x": 100, "y": 300, "area": 5000, - "semantic_string": "description: 'Post image', id context: 'row feed photo imageview'", - "resource_id": "row_feed_photo_imageview", - "original_attribs": {"desc": "Post image", "text": ""} - } - - nodes = [author_node, image_node] - - # The exact string used by _extract_post_content - result = engine._keyword_match_score("post author username header", nodes) - - assert result is not None, "Failed to extract author node via fast path" - assert "fiona.dawson" in result["semantic"], "Extracted wrong node for author" - assert result["score"] >= 0.35, f"Confidence score too low: {result['score']}" - -def test_extract_post_description_confidence(): - """ - Tests that the TelepathicEngine can confidently extract the post description - node from a standard feed XML dump. - """ - engine = TelepathicEngine() - - author_node = { - "x": 100, "y": 200, "area": 500, - "semantic_string": "description: 'fiona.dawson', id context: 'row feed photo profile name'", - "resource_id": "row_feed_photo_profile_name", - "original_attribs": {"desc": "fiona.dawson", "text": "fiona.dawson"} - } - - image_node = { - "x": 100, "y": 300, "area": 5000, - "semantic_string": "description: 'Post image', id context: 'row feed photo imageview'", - "resource_id": "row_feed_photo_imageview", - "original_attribs": {"desc": "Post image", "text": ""} - } - - nodes = [author_node, image_node] - - # The exact string used by _extract_post_content - result = engine._keyword_match_score("post image video media content description", nodes) - - assert result is not None, "Failed to extract image/media node via fast path" - assert "imageview" in result["semantic"], "Extracted wrong node for media" - assert result["score"] >= 0.35, f"Confidence score too low: {result['score']}"