mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(sae): stabilize navigation engine, fix container filtering, and negative reinforcement logic

Browse Source
This commit is contained in:
Marc Mintel
2026-04-25 13:09:12 +02:00
parent ad1af4edfe
commit 77e8251aa7
43 changed files with 4107 additions and 4075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
GramAddict/__main__.py
View File

@@ -1,8 +1,8 @@
from GramAddict.core.agentic_views import *
import argparse
from os import getcwd, path
from GramAddict import __version__
from GramAddict.core.agentic_views import *
from GramAddict.core.bot_flow import start_bot
from GramAddict.core.download_from_github import download_from_github
@@ -13,9 +13,7 @@ def cmd_init(args):
for username in args.account_name:
if not path.exists("./run.py"):
print("Creating run.py ...")
download_from_github(
"https://github.com/GramAddict/bot/blob/master/run.py"
)
download_from_github("https://github.com/GramAddict/bot/blob/master/run.py")
if not path.exists(f"./accounts/{username}"):
print(
f"Creating 'accounts/{username}' folder with a config starting point inside. You have to edit these files according with https://docs.gramaddict.org/#/configuration"
@@ -53,8 +51,10 @@ def cmd_dump(args):
os.popen("adb shell pkill atx-agent").close()
try:
d = u2.connect(args.device)
except RuntimeError as err:
raise SystemExit(err)
except Exception as err:
raise SystemExit(
f"⚠️ [ADB ConnectError] Could not connect to device: {err}\nPlease check if ADB is running and your device is authorized."
)
def dump_hierarchy(device, path):
xml_dump = device.dump_hierarchy()
@@ -71,11 +71,7 @@ def cmd_dump(args):
dump_hierarchy(d, "dump/cur/hierarchy.xml")
archive_name = int(time.time())
make_archive(archive_name)
print(
Fore.GREEN
+ Style.BRIGHT
+ "\nCurrent screen dump generated successfully! Please, send me this file:"
)
print(Fore.GREEN + Style.BRIGHT + "\nCurrent screen dump generated successfully! Please, send me this file:")
print(Fore.BLUE + Style.BRIGHT + f"{os.getcwd()}\\screen_{archive_name}.zip")
@@ -126,9 +122,7 @@ def main() -> None:
prog="GramAddict",
description="free human-like Instagram bot",
)
parser.add_argument(
"-v", "--version", action="version", version=f"{parser.prog} {__version__}"
)
parser.add_argument("-v", "--version", action="version", version=f"{parser.prog} {__version__}")
subparser = parser.add_subparsers(dest="subparser")
actions = {}
for c in _commands:

2
GramAddict/core/bot_flow.py
View File

@@ -849,6 +849,7 @@ def _run_zero_latency_feed_loop(
consecutive_marker_misses += 1
if consecutive_marker_misses >= 3:
logger.error("❌ Lost context completely. Aborting feed loop to force reset.")
sae.unlearn_current_state(context_xml)
dump_ui_state(device, "context_lost", {"feed": job_target, "misses": consecutive_marker_misses})
return "CONTEXT_LOST"
@@ -888,6 +889,7 @@ def _run_zero_latency_feed_loop(
consecutive_marker_misses += 1
if consecutive_marker_misses >= 3:
logger.error("❌ Lost context completely. Aborting feed loop to force reset.")
sae.unlearn_current_state(context_xml)
dump_ui_state(device, "context_lost", {"feed": job_target, "misses": consecutive_marker_misses})
return "CONTEXT_LOST"

98
GramAddict/core/device_facade.py
View File

@@ -1,19 +1,17 @@
import logging
import json
import os
import re
import uiautomator2 as u2
from time import sleep, time
from random import uniform
from GramAddict.core.utils import random_sleep
from functools import wraps
from random import uniform
from time import sleep
from GramAddict.core.physics.biomechanics import PhysicsBody, BezierGesture
import uiautomator2 as u2
from GramAddict.core.physics.biomechanics import BezierGesture, PhysicsBody
from GramAddict.core.physics.sendevent_injector import SendEventInjector
logger = logging.getLogger(__name__)
def adb_retry(retries=3, delay=2.0):
def decorator(func):
@wraps(func)
@@ -28,18 +26,38 @@ def adb_retry(retries=3, delay=2.0):
sleep(delay * (attempt + 1)) # Exponential backoff
logger.error(f"❌ ADB action {func.__name__} failed after {retries} retries. Crashing gracefully.")
raise last_err
return wrapper
return decorator
def create_device(device_id, app_id, args=None):
try:
return DeviceFacade(device_id, app_id, args)
except Exception as e:
err_str = str(e)
err_type = str(type(e))
if (
"ConnectError" in err_type
or "ConnectionRefusedError" in err_type
or "ConnectionError" in err_type
or "Timeout" in err_type
):
logger.error(f"⚠️ [ADB ConnectError] Could not connect to device '{device_id}'.")
logger.error("👉 Please verify:")
logger.error(" 1. Your phone is connected via USB or Wi-Fi.")
logger.error(" 2. 'USB Debugging' is enabled in Developer Options.")
logger.error(" 3. You have authorized this computer on your phone's screen.")
logger.error(" 4. The adb server is running ('adb devices').")
raise SystemExit(1)
logger.error(f"Failed to create device: {e}")
# We don't want to just return None and crash later.
# We don't want to just return None and crash later.
# We should raise so the orchestrator knows it's a fatal boot error.
raise e
def get_device_info(device):
if not device or not device.deviceV2:
logger.error("Cannot get device info: Device not initialized.")
@@ -47,32 +65,29 @@ def get_device_info(device):
info = device.info
logger.debug(f"Device Info: {info.get('productName')} | SDK: {info.get('sdkInt')}")
class DeviceFacade:
deviceV2 = None
app_id = None
device_id = None
def __init__(self, device_id, app_id, args):
self.device_id = device_id
self.app_id = app_id
self.args = args
self.deviceV2 = u2.connect(device_id)
# Configure uiautomator2
self.deviceV2.settings["wait_timeout"] = 3.0
self.deviceV2.settings["post_delay"] = 0.5
# System dialog handler (language-agnostic via resource-id, not text)
try:
# u2 v3.x: named watchers with xpath selectors
# android:id/aerr_close = App crash "Close" button (all languages)
self.deviceV2.watcher("crash_dialog").when(
xpath='//*[@resource-id="android:id/aerr_close"]'
).click()
self.deviceV2.watcher("crash_dialog").when(xpath='//*[@resource-id="android:id/aerr_close"]').click()
# android:id/button1 = positive system dialog button (all languages)
self.deviceV2.watcher("system_dialog").when(
xpath='//*[@resource-id="android:id/button1"]'
).click()
self.deviceV2.watcher("system_dialog").when(xpath='//*[@resource-id="android:id/button1"]').click()
self.deviceV2.watcher.start()
except Exception as e:
logger.debug(f"Could not start system watcher: {e}")
@@ -88,7 +103,7 @@ class DeviceFacade:
@adb_retry()
def cm_to_pixels(self, cm: float) -> int:
info = self.deviceV2.info
dpx = info.get("displaySizeDpX", 400)
dpx = info.get("displaySizeDpX", 400)
width = info.get("displayWidth", 1080)
# Android baseline: 1 dp = 1/160 inch. 1 inch = 2.54 cm
# PPCM (Pixels Per CM) = (width / dpx) * (160 / 2.54)
@@ -106,7 +121,6 @@ class DeviceFacade:
def unlock(self):
self.deviceV2.unlock()
@property
def info(self):
return self.deviceV2.info
@@ -132,7 +146,8 @@ class DeviceFacade:
def swipe(self, sx, sy, ex, ey, duration=None):
"""Pass-through strictly for non-biological bezier swiping (e.g., darwin_engine noise correction)"""
kwargs = {}
if duration is not None: kwargs["duration"] = duration
if duration is not None:
kwargs["duration"] = duration
self.deviceV2.swipe(sx, sy, ex, ey, **kwargs)
@adb_retry()
@@ -146,14 +161,14 @@ class DeviceFacade:
@adb_retry()
def click(self, x=None, y=None, obj=None):
if obj:
if isinstance(obj, dict) and 'x' in obj and 'y' in obj:
self.human_click(obj['x'], obj['y'])
if isinstance(obj, dict) and "x" in obj and "y" in obj:
self.human_click(obj["x"], obj["y"])
return
try:
left, top, right, bottom = obj.bounds()
w = right - left
h = bottom - top
# Biological fingerprint via PhysicsBody
body = PhysicsBody.get_session_instance(self)
# Thumb bias: right-handers land slightly left-below center
@@ -163,20 +178,21 @@ class DeviceFacade:
else:
cx_base = left + (w * 0.55)
cy_base = top + (h * 0.55)
from random import gauss
# Fatigue increases spread
fatigue_mult = 1.0 + body.fatigue * 0.3
sigma_x = max(1, w * 0.15 * fatigue_mult)
sigma_y = max(1, h * 0.15 * fatigue_mult)
cx = int(gauss(cx_base, sigma_x))
cy = int(gauss(cy_base, sigma_y))
# Math constraint to ensure it physically lands on the button
cx = max(left + 1, min(cx, right - 1))
cy = max(top + 1, min(cy, bottom - 1))
self.human_click(cx, cy)
except Exception as e:
logger.debug(f"Bounds extraction failed, fallback to native click: {e}")
@@ -193,7 +209,6 @@ class DeviceFacade:
self.deviceV2.shell(f"input tap {int(x)} {int(y)}")
return
from random import uniform
try:
body = PhysicsBody.get_session_instance(self)
injector = SendEventInjector.get_instance(self)
@@ -201,7 +216,6 @@ class DeviceFacade:
tap_duration = uniform(40, 90)
timing = BezierGesture.compute_sigmoid_timing(len(points), tap_duration)
injector.inject_gesture(points, timing, touch_major=body.get_touch_major())
except Exception as e:
logger.debug(f"human_click biomechanics failed, fallback: {e}")
@@ -234,18 +248,17 @@ class DeviceFacade:
try:
body = PhysicsBody.get_session_instance(self)
injector = SendEventInjector.get_instance(self)
# Use scroll_curve for vertical swipes, horizontal_swipe_curve for horizontal
is_horizontal = abs(end_x - start_x) > abs(end_y - start_y)
if is_horizontal:
points = BezierGesture.horizontal_swipe_curve((start_x, start_y), (end_x, end_y), body)
else:
points = BezierGesture.scroll_curve((start_x, start_y), (end_x, end_y), body)
# Use fling timing (J-curve) to ensure high terminal velocity so Android scroll physics works natively
timing = BezierGesture.compute_fling_timing(len(points), dur_ms)
injector.inject_gesture(points, timing, touch_major=body.get_touch_major())
except Exception as e:
logger.debug(f"human_swipe biomechanics failed, fallback to native swipe: {e}")
@@ -263,14 +276,14 @@ class DeviceFacade:
pkg = self.deviceV2.app_current().get("package")
if pkg == self.app_id:
return pkg
# Brief retry: many false positives come from <500ms notification banners
# A single short wait handles ALL transient overlays regardless of source app
sleep(0.5)
pkg = self.deviceV2.app_current().get("package")
# If still not our app, check if it's just SystemUI (always present, never a real takeover)
if pkg in ('com.android.systemui', 'android'):
if pkg in ("com.android.systemui", "android"):
return self.app_id
return pkg
@@ -284,38 +297,41 @@ class DeviceFacade:
def dump_hierarchy(self):
# Compressed=True dramatically speeds up UIAutomator2 dumps by skipping invisible elements!
xml = self.deviceV2.dump_hierarchy(compressed=True)
# Continuous Session Tracing
from datetime import datetime
try:
if not hasattr(self, "_trace_counter"):
self._trace_counter = 0
ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self._trace_dir = os.path.join("debug", "session_traces", ts)
os.makedirs(self._trace_dir, exist_ok=True)
self._trace_counter += 1
trace_path = os.path.join(self._trace_dir, f"{self._trace_counter:05d}.xml")
with open(trace_path, "w", encoding="utf-8") as f:
f.write(xml)
except Exception as e:
logger.debug(f"Failed to write session trace: {e}")
return xml
@adb_retry()
def get_screenshot_b64(self):
import base64
from io import BytesIO
img = self.deviceV2.screenshot()
buffered = BytesIO()
img.save(buffered, format="JPEG", quality=70) # Compressed for target latency
return base64.b64encode(buffered.getvalue()).decode('utf-8')
img.save(buffered, format="JPEG", quality=70) # Compressed for target latency
return base64.b64encode(buffered.getvalue()).decode("utf-8")
# Telepathic Semantic UI Integration
@adb_retry()
def find_semantic(self, intent_description: str):
from GramAddict.core.telepathic_engine import TelepathicEngine
engine = TelepathicEngine.get_instance()
xml = self.dump_hierarchy()
# Passing self (DeviceFacade) enables the Vision Cortex VLM fallback

849
GramAddict/core/goap.py
View File

@@ -13,858 +13,25 @@ Like a GPS navigation system:
- It remembers shortcuts (learn)
"""
import hashlib
import logging
import re
import time
import xml.etree.ElementTree as ET
from enum import Enum
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List
from GramAddict.core.qdrant_memory import QdrantBase
from GramAddict.core.utils import random_sleep
logger = logging.getLogger(__name__)
# ══════════════════════════════════════════════════════
# 1. SCREEN IDENTITY — "Where am I?"
# ══════════════════════════════════════════════════════
class ScreenType(Enum):
HOME_FEED = "home_feed"
EXPLORE_GRID = "explore_grid"
REELS_FEED = "reels_feed"
OWN_PROFILE = "own_profile"
OTHER_PROFILE = "other_profile"
POST_DETAIL = "post_detail"
STORY_VIEW = "story_view"
DM_INBOX = "dm_inbox"
DM_THREAD = "dm_thread"
SEARCH_RESULTS = "search_results"
FOLLOW_LIST = "follow_list"
COMMENTS = "comments"
MODAL = "modal"
FOREIGN_APP = "foreign_app"
UNKNOWN = "unknown"
class ScreenIdentity:
"""
Understands what screen the bot is on by analyzing the XML dump.
NO hardcoded states — purely structural analysis.
This is the bot's EYES. It answers: "What do I see right now?"
"""
def __init__(self, bot_username: str):
self.bot_username = bot_username.lower()
try:
from GramAddict.core.qdrant_memory import ScreenMemoryDB
self.screen_memory = ScreenMemoryDB()
except ImportError:
self.screen_memory = None
def identify(self, xml_dump: str) -> Dict[str, Any]:
"""
Analyzes an XML dump and returns a complete screen description.
Returns:
{
'screen_type': ScreenType,
'available_actions': ['tap like button', 'tap explore tab', ...],
'selected_tab': 'feed_tab' | 'search_tab' | ...,
'context': {'username': '...', 'post_count': '...', ...}
}
"""
if not xml_dump or not isinstance(xml_dump, str):
return self._empty_screen()
try:
clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip()
root = ET.fromstring(clean)
except Exception:
return self._empty_screen()
# Extract structural signals
packages = set()
resource_ids = set()
content_descs = []
texts = []
selected_tab = None
clickable_elements = []
app_id = "com.instagram.android"
for elem in root.iter("node"):
pkg = elem.get("package", "")
if pkg:
packages.add(pkg)
rid = elem.get("resource-id", "").strip()
text = elem.get("text", "").strip()
desc = elem.get("content-desc", "").strip()
clickable = elem.get("clickable", "false") == "true"
selected = elem.get("selected", "false") == "true"
bounds = elem.get("bounds", "")
if rid:
# Normalize: "com.instagram.android:id/feed_tab" → "feed_tab"
short_id = rid.split("/")[-1] if "/" in rid else rid
resource_ids.add(short_id)
# Track which tab is selected
if selected and short_id in ("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab"):
selected_tab = short_id
if text:
texts.append(text)
if desc:
content_descs.append(desc)
if clickable and bounds:
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
if match:
left, t, r, b = map(int, match.groups())
cx, cy = (left + r) // 2, (t + b) // 2
clickable_elements.append(
{
"text": text,
"desc": desc,
"id": rid.split("/")[-1] if "/" in rid else rid,
"x": cx,
"y": cy,
"bounds": bounds,
}
)
# ── Foreign app check ──
if app_id not in packages:
return {
"screen_type": ScreenType.FOREIGN_APP,
"available_actions": ["press back", "force start instagram"],
"selected_tab": None,
"context": {"packages": list(packages)},
"signature": self._compute_signature(resource_ids, content_descs, texts),
}
desc_lower = " ".join(content_descs).lower()
text_lower = " ".join(texts).lower()
ids_str = " ".join(resource_ids).lower()
signature = self._compute_signature(resource_ids, content_descs, texts)
# ── Identify screen type from structural signals ──
screen_type = self._classify_screen(
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature
)
# ── Extract available actions from clickable elements ──
available_actions = self._extract_available_actions(
clickable_elements, resource_ids, content_descs, texts, screen_type
)
# ── Extract context ──
context = self._extract_context(content_descs, texts, resource_ids, screen_type)
return {
"screen_type": screen_type,
"available_actions": available_actions,
"selected_tab": selected_tab,
"context": context,
"signature": signature,
}
def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None):
"""Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
# Priority 0: Content-creation overlays that block ALL navigation.
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
if any(marker in ids_str for marker in creation_flow_markers):
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
return ScreenType.MODAL
# Priority 1: Check Qdrant Semantic Cache
if signature and self.screen_memory and self.screen_memory.is_connected:
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
if cached_type_str:
try:
return ScreenType[cached_type_str]
except KeyError:
pass
# Priority 2: Structural Heuristics (Instant, for core tabs)
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
return ScreenType.FOLLOW_LIST
if "profile_header_container" in ids:
return ScreenType.OTHER_PROFILE
# Reels structural markers — present even when Instagram hides the tab bar
# in full-screen Reels viewing. Without this, selected_tab=None → UNKNOWN.
REELS_MARKERS = ("clips_viewer_container", "root_clips_layout", "clips_linear_layout_container")
if any(marker in ids for marker in REELS_MARKERS):
return ScreenType.REELS_FEED
# DM thread detection — structural markers present inside DM conversations
if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids:
return ScreenType.DM_THREAD
if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab:
return ScreenType.POST_DETAIL
if selected_tab == "feed_tab":
return ScreenType.HOME_FEED
if selected_tab == "clips_tab":
return ScreenType.REELS_FEED
if selected_tab == "search_tab":
return ScreenType.EXPLORE_GRID
if selected_tab == "profile_tab":
return ScreenType.OWN_PROFILE
if selected_tab == "direct_tab":
return ScreenType.DM_INBOX
if "message_input" in ids:
return ScreenType.DM_INBOX # Fallback for DM thread as inbox
# Priority 3: Semantic VLM Classification Fallback
from GramAddict.core.config import Config
from GramAddict.core.llm_provider import query_llm
cfg = Config()
url = (
getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat")
if hasattr(cfg, "args")
else "http://localhost:11434/api/chat"
)
model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3"
layout_context = (
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
)
prompt = (
f"Identify the Instagram screen layout type based on these DOM structural signals.\n"
f"Valid types: {[t.name for t in ScreenType]}\n"
f"Context:\n{layout_context}\n"
f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches."
)
try:
response = query_llm(
url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False
)
if response and isinstance(response, str):
result = response.strip().upper()
elif response and isinstance(response, dict) and "response" in response:
result = response["response"].strip().upper()
else:
return ScreenType.UNKNOWN
for t in ScreenType:
if t.name in result:
if signature and self.screen_memory:
self.screen_memory.store_screen(signature, t.name)
return t
except Exception as e:
import logging
logging.getLogger(__name__).debug(f"LLM Classification failed: {e}")
return ScreenType.UNKNOWN
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type):
"""Discover what actions are possible on this screen."""
actions = []
# Navigation tabs (always available when visible)
tab_map = {
"feed_tab": "tap home tab",
"search_tab": "tap explore tab",
"clips_tab": "tap reels tab",
"profile_tab": "tap profile tab",
"direct_tab": "tap messages tab",
}
for tab_id, action in tab_map.items():
if tab_id in resource_ids:
actions.append(action)
# Screen-specific actions
desc_lower = " ".join(content_descs).lower()
text_lower = " ".join(texts).lower()
if "like" in desc_lower:
actions.append("tap like button")
if "comment" in desc_lower:
actions.append("tap comment button")
if "share" in desc_lower:
actions.append("tap share button")
if "save" in desc_lower or "bookmark" in desc_lower:
actions.append("tap save button")
if "back" in desc_lower:
actions.append("tap back button")
if any("follow" in e.get("text", "").lower() for e in clickable_elements):
actions.append("tap follow button")
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
if "message" in desc_lower or "nachricht" in desc_lower:
actions.append("tap message button")
if (
"following" in desc_lower
or "abonniert" in desc_lower
or "following" in text_lower
or "profile_header_following" in " ".join(resource_ids).lower()
):
actions.append("tap following list")
# Grid items
if screen_type == ScreenType.EXPLORE_GRID:
actions.append("tap first grid item")
# Scroll
actions.append("scroll down")
actions.append("press back")
return list(set(actions)) # Deduplicate
def _extract_context(self, content_descs, texts, resource_ids, screen_type):
"""Extract meaningful context from the screen."""
context = {}
desc_text = " ".join(content_descs)
# Username on profile
username_match = re.search(r"(\w+)'s (?:profile|story|unseen story)", desc_text)
if username_match:
context["username"] = username_match.group(1)
# Post/follower counts
for d in content_descs:
m = re.match(r"([\d,.]+K?M?)(\s*)(posts?|followers?|following)", d, re.IGNORECASE)
if m:
context[m.group(3).lower()] = m.group(1)
# Like state
for d in content_descs:
if d.lower() == "liked":
context["is_liked"] = True
elif d.lower() == "like":
context["is_liked"] = False
return context
def _compute_signature(self, resource_ids, content_descs, texts):
"""Compute a stable hash for this screen state (for Qdrant lookup)."""
# Use sorted IDs + key content for stability
sig_parts = sorted(resource_ids)[:20]
sig_parts.extend(sorted(set(d.lower()[:30] for d in content_descs if len(d) > 2))[:10])
sig = "|".join(sig_parts)
return hashlib.sha256(sig.encode()).hexdigest()[:24]
def _empty_screen(self):
return {
"screen_type": ScreenType.FOREIGN_APP,
"available_actions": ["press back", "force start instagram"],
"selected_tab": None,
"context": {},
"signature": "empty",
}
from GramAddict.core.navigation.knowledge import NavigationKnowledge
from GramAddict.core.navigation.path_memory import PathMemory
from GramAddict.core.navigation.planner import GoalPlanner
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
# Re-export for backward compatibility (optional but helps minimize import breakage)
__all__ = ["GoalExecutor", "ScreenIdentity", "ScreenType", "PathMemory", "NavigationKnowledge", "GoalPlanner"]
# ══════════════════════════════════════════════════════
# 2. PATH MEMORY — "How did I get there last time?"
# ══════════════════════════════════════════════════════
class PathMemory:
"""
Qdrant-backed memory for successful navigation paths.
Stores: goal → [step1, step2, ...] → success
Enables instant recall for known goals.
"""
def __init__(self, username: str = ""):
self.username = username
try:
suffix = f"_{username}" if username else ""
self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768)
except Exception:
self._db = None
def wipe(self):
"""Wipe all learned navigation paths from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.wipe_collection()
except Exception as e:
logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}")
def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]:
"""
Recall a previously successful path for this goal from this screen type.
Returns list of steps or None.
"""
if not self._db or not self._db.is_connected:
return None
query = f"goal: {goal} | from: {current_screen_type}"
vec = self._db._get_embedding(query)
if not vec:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.query_points(
collection_name=self._db.collection_name,
query=vec,
query_filter=Filter(
must=[FieldCondition(key="start_screen", match=MatchValue(value=current_screen_type))]
),
limit=3,
score_threshold=0.85,
).points
for r in results:
p = r.payload
if p.get("success") and p.get("steps"):
logger.info(
f"🧠 [GOAP Recall] Found path for '{goal}': "
f"{len(p['steps'])} steps (confidence: {p.get('confidence', 0):.2f})"
)
return p["steps"]
return None
except Exception as e:
logger.debug(f"GOAP recall error: {e}")
return None
def learn_path(self, goal: str, start_screen: str, steps: List[Dict], success: bool):
"""Store a navigation path in Qdrant."""
if not self._db or not self._db.is_connected:
return
query = f"goal: {goal} | from: {start_screen}"
vec = self._db._get_embedding(query)
if not vec:
return
seed = f"{goal}|{start_screen}"
payload = {
"goal": goal,
"start_screen": start_screen,
"steps": steps,
"step_count": len(steps),
"success": success,
"confidence": 0.85 if success else 0.0,
"timestamp": time.time(),
}
outcome = "✅" if success else "❌"
self._db.upsert_point(
seed,
payload,
vector=vec,
log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}",
)
def forget_path(self, goal: str, start_screen: str):
"""Remove a cached path to force re-discovery."""
if not self._db or not self._db.is_connected:
return
seed = f"{goal}|{start_screen}"
try:
from qdrant_client import models
point_id = self._db._get_id(seed)
self._db.client.delete(
collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id])
)
except Exception as e:
logger.debug(f"Failed to forget path: {e}")
# ══════════════════════════════════════════════════════
# 3. GOAL PLANNER — "What should I do next?"
# ══════════════════════════════════════════════════════
class NavigationKnowledge:
"""
Manages the bot's learned understanding of the Instagram UI.
Discovered dynamically through exploration and success.
"""
def __init__(self, username: str):
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
# In-memory cache for rapidly avoiding traps during exploration
# In-memory cache for rapidly avoiding traps during exploration
self._learned_screen_mappings = {}
self._learned_traps = set()
def wipe(self):
"""Wipe all learned knowledge from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.wipe_collection()
except Exception as e:
logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}")
def update_username(self, username: str):
"""Update username and reconnect DB if needed."""
if self.username != username:
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
def get_requirements(self, goal: str) -> List[ScreenType]:
"""Get required screens for a goal. Returns known requirements or empty list."""
if not self._db or not self._db.is_connected:
return []
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="goal", match=MatchValue(value=goal))]),
limit=1,
)[0]
if results:
screen_name = results[0].payload.get("required_screen")
logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}")
if screen_name:
return [ScreenType[screen_name]]
except Exception as e:
logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}")
return []
def learn_goal_requirement(self, goal: str, screen_type: ScreenType):
"""Learn that achieving 'goal' lands us on 'screen_type'."""
if not self._db or not self._db.is_connected:
logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected")
return
seed = f"req_{goal}"
vec = self._db._get_embedding(f"goal_requirement: {goal}")
payload = {"goal": goal, "required_screen": screen_type.name, "timestamp": time.time()}
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}' → {screen_type.name}")
def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]:
"""Find which action leads to this screen."""
for action, screen in self._learned_screen_mappings.items():
if screen == target_screen:
return action
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[FieldCondition(key="result_screen", match=MatchValue(value=target_screen.name))]
),
limit=1,
)[0]
if results:
return results[0].payload.get("action")
except Exception:
pass
return None
def get_screen_for_action(self, action: str) -> Optional[ScreenType]:
"""Find where this action leads to to avoid looping traps."""
if action in self._learned_screen_mappings:
return self._learned_screen_mappings[action]
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="action", match=MatchValue(value=action))]),
limit=1,
)[0]
if results:
screen_name = results[0].payload.get("result_screen")
if screen_name:
return ScreenType[screen_name]
except Exception:
pass
return None
def learn_screen_mapping(self, action: str, result_screen: ScreenType):
"""Learn that taking 'action' leads to 'result_screen'."""
if not self._db or not self._db.is_connected:
return
seed = f"map_{action}"
vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}")
payload = {"action": action, "result_screen": result_screen.name, "timestamp": time.time()}
self._learned_screen_mappings[action] = result_screen
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}' → {result_screen.name}")
def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]:
"""Find where this tab leads to to avoid looping traps."""
if tab_id in self._learned_screen_mappings:
return self._learned_screen_mappings[tab_id]
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="tab_id", match=MatchValue(value=tab_id))]),
limit=1,
)[0]
if results:
s_name = results[0].payload.get("result_screen")
if s_name:
return ScreenType[s_name]
except Exception:
pass
return None
def learn_trap(self, screen_type: ScreenType, action: str, trap_reason: str = "softlock"):
"""Aversively learn that an action on a screen is dangerous/useless."""
trap_key = f"{screen_type.name}_{action}"
self._learned_traps.add(trap_key)
if not self._db or not self._db.is_connected:
return
seed = f"trap_{trap_key}"
# Aversive vector is completely orthogonal to normal goals to prevent retrieval overlap
vec = self._db._get_embedding(f"trap_avoidance: {trap_key} {trap_reason}")
payload = {
"trap_screen": screen_type.name,
"trap_action": action,
"trap_reason": trap_reason,
"timestamp": time.time(),
}
self._db.upsert_point(seed, payload, vector=vec)
logger.error(f"💀 [Aversive Learning] BURNED action '{action}' on {screen_type.name} due to: {trap_reason}")
def is_trap(self, screen_type: ScreenType, action: str) -> bool:
"""Check if an action on this screen is a known trap."""
trap_key = f"{screen_type.name}_{action}"
if trap_key in self._learned_traps:
return True
if not self._db or not self._db.is_connected:
return False
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(key="trap_screen", match=MatchValue(value=screen_type.name)),
FieldCondition(key="trap_action", match=MatchValue(value=action)),
]
),
limit=1,
)[0]
if results:
self._learned_traps.add(trap_key)
return True
except Exception:
pass
return False
class GoalPlanner:
"""
Given a goal and current screen state, plans the next action.
Uses Dynamic Discovery to navigate without hardcoded maps.
"""
def __init__(self, username: str):
self.knowledge = NavigationKnowledge(username)
def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]:
"""Plans the NEXT single action to take toward the goal."""
screen_type = screen["screen_type"]
available = screen.get("available_actions", [])
context = screen.get("context", {})
goal_lower = goal.lower()
# ── 1. Check if goal is ALREADY achieved ──
if self._is_goal_achieved(goal_lower, screen_type, context):
logger.info(f"🎯 [GOAP] Goal '{goal}' already achieved on {screen_type.value}!")
return None
# (Phase 5: legacy _plan_goal_action static heuristics purged,
# all intents fall through to VLM-driven Discovery in _plan_navigation)
# ── 3. Am I on the right screen? If not, navigate there ──
selected_tab = screen.get("selected_tab")
nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions)
if nav_action:
return nav_action
# Final fallback: back-track, UNLESS back-tracking is a known trap on this screen!
if not self.knowledge.is_trap(screen_type, "press back"):
return "press back"
# We are trapped! Can't go forward, can't go back!
logger.error(f"💀 [GOAP] Completely trapped on {screen_type.name}. Forcing Instagram restart.")
return "force start instagram"
def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool:
"""Check if the goal is already satisfied. Delegates to ScreenTopology SSOT."""
from GramAddict.core.screen_topology import ScreenTopology
# Interaction goals (context-specific, not navigation)
if "like" in goal and context.get("is_liked") is True:
return True
if "view profile" in goal and screen_type in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE):
return True
# Navigation goals — delegate to SSOT
target = ScreenTopology.goal_to_target_screen(goal)
if target and screen_type == target:
return True
return False
def _plan_navigation(
self,
goal: str,
screen_type: ScreenType,
available: List[str],
selected_tab: Optional[str] = None,
explored_nav_actions: set = None,
) -> Optional[str]:
"""If we're on the wrong screen, figure out how to navigate.
Strategy (priority order):
1. HD Map (ScreenTopology BFS) — deterministic, pre-computed routes
2. Learned Knowledge (Qdrant) — dynamic discovery from past sessions
3. Autonomous Discovery — linguistic matching + VLM intent
"""
from GramAddict.core.screen_topology import ScreenTopology
# 0. Aversive Filter: Remove known traps from available actions
safe_available = []
for action in available:
if not self.knowledge.is_trap(screen_type, action):
safe_available.append(action)
else:
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
available = safe_available
# ── 1. HD Map Routing (Primary Strategy) ──
target_screen = ScreenTopology.goal_to_target_screen(goal)
if target_screen and target_screen != screen_type:
route = ScreenTopology.find_route(screen_type, target_screen)
if route:
next_action, next_screen = route[0]
# Verify action isn't explored/trapped
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
route_desc = " → ".join(s.name for _, s in route)
logger.info(
f"🗺️ [HD Map] Route: {screen_type.name} → {route_desc}. " f"Next action: '{next_action}'"
)
return next_action
else:
logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.")
else:
logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.")
# ── 2. Learned Knowledge (Qdrant) ──
required_screens = self.knowledge.get_requirements(goal)
# ── 3. Autonomous Discovery (Blank Start fallback) ──
if not required_screens:
logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.")
# Return raw intent for TelepathicEngine discovery (VLM)
if explored_nav_actions and goal in explored_nav_actions:
logger.info(
f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking."
)
return None # Don't return goal again — force fallback to press back
else:
return goal
# 4. If we're already on an acceptable screen, no navigation needed
if screen_type in required_screens:
return None
# 5. Find the action we need to take (from learned knowledge or HD map)
for target_screen in required_screens:
# Try HD Map first!
route = ScreenTopology.find_route(screen_type, target_screen)
if route:
next_action, next_screen = route[0]
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
logger.info(f"🧭 [Nav HD Map] Routing to required {target_screen.name} via '{next_action}'")
return next_action
known_action = self.knowledge.get_action_for_screen(target_screen)
if not known_action:
logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...")
screen_friendly_name = target_screen.name.replace("_", " ").lower()
goal_words = [w.rstrip("s") for w in screen_friendly_name.split() if len(w) > 3]
for action in available:
if any(w in action.lower() for w in goal_words):
known_target = self.knowledge.get_screen_for_action(action)
if known_target and known_target != target_screen:
continue
logger.info(
f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'"
)
return action
return f"navigate to {screen_friendly_name}"
else:
if known_action in available:
logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'")
return known_action
# If no targeted navigation works, try going back first
if "press back" in available:
return "press back"
return None
# ══════════════════════════════════════════════════════
# 4. GOAL EXECUTOR — The Main Brain Loop
# GOAL EXECUTOR — The Main Brain Loop
# ══════════════════════════════════════════════════════

216
GramAddict/core/llm_provider.py
View File

@@ -1,71 +1,77 @@
import re
import os
import json
import requests
import logging
from typing import Optional, List, Dict
import os
import re
from typing import List, Optional
import requests
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
logger = logging.getLogger(__name__)
def extract_json(text: str) -> Optional[str]:
"""
Robustly extracts the first JSON object or array from a string that may contain
Robustly extracts the first JSON object or array from a string that may contain
natural language prefix/suffix. Also purges <think> blocks and markdown ticks.
"""
if not text:
return None
# 100% Autonomous: Scrub model's internal thinking process
if "<think>" in text:
text = re.sub(r'<think>.*?</think>', '', text, flags=re.DOTALL).strip()
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
logger.debug("🧠 [LLM] Scoped thinking block detected and purged.")
# Remove markdown code block formats
text = re.sub(r'^```json\s*', '', text, flags=re.MULTILINE)
text = re.sub(r'^```\s*', '', text, flags=re.MULTILINE)
text = re.sub(r"^```json\s*", "", text, flags=re.MULTILINE)
text = re.sub(r"^```\s*", "", text, flags=re.MULTILINE)
# Try perfect json block extraction first
match = re.search(r'(\{.*\}|\[.*\])', text, re.DOTALL)
match = re.search(r"(\{.*\}|\[.*\])", text, re.DOTALL)
if match:
candidate = match.group(0)
try:
import json
json.loads(candidate)
return candidate
except Exception:
pass
# Smart Fallback: Truncated JSON Healing
# If standard validation fails (e.g., due to EOF truncation by local models),
# run a regex extraction pass over the raw generated text to safely salvage
# If standard validation fails (e.g., due to EOF truncation by local models),
# run a regex extraction pass over the raw generated text to safely salvage
# all key-value pairs that *were* successfully completed before the truncation.
import json
matches = re.findall(r'"([a-zA-Z0-9_]+)"\s*:\s*(?:([0-9.-]+)|"([^"\\]*(?:\\.[^"\\]*)*)")', text)
if matches:
res = {}
for k, num, obj in matches:
if num:
try:
res[k] = float(num) if '.' in num else int(num)
res[k] = float(num) if "." in num else int(num)
except ValueError:
res[k] = num
else:
res[k] = obj.replace('\\"', '"')
recovered_json = json.dumps(res)
logger.warning(f"🔧 [Fuzzy Parse] Successfully salvaged {len(res)} keys from heavily truncated LLM output.")
return recovered_json
return None
_MODEL_PRICING_CACHE = None
def get_model_pricing(model_id: str) -> dict:
global _MODEL_PRICING_CACHE
if _MODEL_PRICING_CACHE is None:
@@ -78,118 +84,128 @@ def get_model_pricing(model_id: str) -> dict:
_MODEL_PRICING_CACHE = {}
except Exception:
_MODEL_PRICING_CACHE = {}
# Check if exact match exists, if not, try partial matches (e.g., if version suffixes differ)
if _MODEL_PRICING_CACHE and model_id not in _MODEL_PRICING_CACHE:
for k, v in _MODEL_PRICING_CACHE.items():
if model_id in k or k in model_id:
return v
return _MODEL_PRICING_CACHE.get(model_id, {})
def prewarm_ollama_models(configs):
"""
Sends a dummy request to the configured local Ollama API endpoints via a background thread
Sends a dummy request to the configured local Ollama API endpoints via a background thread
to force the models to load into VRAM during bot startup, minimizing initial connection latency
and avoiding timeouts downstream.
"""
args = configs.args
def _warmup():
import threading
models_to_warm = set()
# Collect unique local models
for attr, url_attr in [
("ai_telepathic_model", "ai_telepathic_url"),
("ai_fallback_model", "ai_fallback_url"),
("ai_condenser_model", "ai_condenser_url"),
("ai_model", "ai_model_url")
("ai_model", "ai_model_url"),
]:
url = getattr(args, url_attr, "")
model = getattr(args, attr, "")
if model and url and ("localhost" in url or "127.0.0.1" in url):
models_to_warm.add((url, model))
for url, model in models_to_warm:
logger.info(f"🔥 [VRAM Pre-Warm] Instructing local Ollama engine to load {model} into memory in the background...")
logger.info(
f"🔥 [VRAM Pre-Warm] Instructing local Ollama engine to load {model} into memory in the background..."
)
try:
# Fire an ultra-short generation to force it into VRAM
requests.post(
url,
json={"model": model, "prompt": "Hi", "stream": False, "options": {"num_predict": 1}},
timeout=120
url,
json={"model": model, "prompt": "Hi", "stream": False, "options": {"num_predict": 1}},
timeout=120,
)
except Exception:
pass
if hasattr(args, "ai_telepathic_model"):
import threading
threading.Thread(target=_warmup, daemon=True).start()
def unload_ollama_models(configs):
"""
Sends keep_alive: 0 to all configured local Ollama API endpoints via a background thread
Sends keep_alive: 0 to all configured local Ollama API endpoints via a background thread
to force the models to unload from VRAM during bot shutdown.
"""
args = configs.args
def _unload():
import threading
models_to_unload = set()
# Collect unique local models
for attr, url_attr in [
("ai_telepathic_model", "ai_telepathic_url"),
("ai_fallback_model", "ai_fallback_url"),
("ai_condenser_model", "ai_condenser_url"),
("ai_model", "ai_model_url")
("ai_model", "ai_model_url"),
]:
url = getattr(args, url_attr, "")
model = getattr(args, attr, "")
if model and url and ("localhost" in url or "127.0.0.1" in url):
models_to_unload.add((url, model))
for url, model in models_to_unload:
logger.info(f"❄️ [VRAM Cleanup] Instructing local Ollama engine to unload {model} from memory...")
try:
# Fire keep_alive: 0 to unload it from VRAM
requests.post(
url,
json={"model": model, "keep_alive": 0},
timeout=5
)
requests.post(url, json={"model": model, "keep_alive": 0}, timeout=5)
except Exception as e:
logger.debug(f"Failed to unload {model}: {e}")
if hasattr(args, "ai_telepathic_model"):
import threading
threading.Thread(target=_unload, daemon=True).start()
def log_openrouter_burn():
"""Fetches and logs the current OpenRouter API key usage (money burned) ONLY if OpenRouter is actively used."""
key = os.environ.get("OPENROUTER_API_KEY")
if not key:
return
try:
from GramAddict.core.config import Config
args = Config().args
uses_openrouter = False
# Check all possible model/url endpoints for 'openrouter'
for attr in ["ai_model", "ai_model_url", "ai_telepathic_model", "ai_telepathic_url",
"ai_fallback_model", "ai_fallback_url", "ai_condenser_model", "ai_condenser_url"]:
for attr in [
"ai_model",
"ai_model_url",
"ai_telepathic_model",
"ai_telepathic_url",
"ai_fallback_model",
"ai_fallback_url",
"ai_condenser_model",
"ai_condenser_url",
]:
val = getattr(args, attr, "")
if val and "openrouter" in str(val).lower():
uses_openrouter = True
break
if not uses_openrouter:
return
except Exception:
pass
try:
r = requests.get("https://openrouter.ai/api/v1/auth/key", headers={"Authorization": f"Bearer {key}"}, timeout=5)
if r.status_code == 200:
@@ -197,11 +213,16 @@ def log_openrouter_burn():
total_spent = data.get("usage", 0.0)
daily_spent = data.get("usage_daily", 0.0)
limit = data.get("limit")
logger.info(f"🔥 [OpenRouter Burn Rate] Daily: ${daily_spent:.4f} | Total: ${total_spent:.4f}" + (f" | Limit: ${limit}" if limit else ""), extra={"color": "\x1b[38;5;208m\x1b[1m"})
logger.info(
f"🔥 [OpenRouter Burn Rate] Daily: ${daily_spent:.4f} | Total: ${total_spent:.4f}"
+ (f" | Limit: ${limit}" if limit else ""),
extra={"color": "\x1b[38;5;208m\x1b[1m"},
)
except Exception as e:
logger.debug(f"Could not fetch OpenRouter burn rate: {e}")
def query_llm(
url: str,
model: str,
@@ -213,16 +234,16 @@ def query_llm(
fallback_model: Optional[str] = None,
fallback_url: Optional[str] = None,
temperature: Optional[float] = None,
max_tokens: Optional[int] = None
max_tokens: Optional[int] = None,
) -> Optional[dict]:
"""
Unified LLM API Caller with configurable fallback.
"""
openrouter_key = os.environ.get("OPENROUTER_API_KEY")
# URL-based provider detection (not model-name based — works for any model)
is_openai_compat = "/v1/chat/completions" in url or "openrouter.ai" in url.lower() or "openai.com" in url.lower()
# If using a cloud model but a local URL was passed, fix it
if not is_openai_compat and ("openrouter" in model.lower() or "/" in model):
# Model looks like "org/model-name" which is OpenRouter format
@@ -230,54 +251,43 @@ def query_llm(
url = "https://openrouter.ai/api/v1/chat/completions"
headers = {"Content-Type": "application/json"}
if is_openai_compat:
if openrouter_key:
headers["Authorization"] = f"Bearer {openrouter_key}"
messages = []
if system:
messages.append({"role": "system", "content": system})
user_content = []
if prompt:
user_content.append({"type": "text", "text": prompt})
if images_b64:
for img in images_b64:
user_content.append({
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{img}"}
})
user_content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img}"}})
messages.append({"role": "user", "content": user_content if len(user_content) > 1 else prompt})
req_data = {
"model": model,
"messages": messages,
"stream": False
}
req_data = {"model": model, "messages": messages, "stream": False}
if format_json:
req_data["response_format"] = {"type": "json_object"}
if temperature is not None:
req_data["temperature"] = temperature
if max_tokens is not None:
req_data["max_tokens"] = max_tokens
else:
# Ollama /generate API
req_data = {
"model": model,
"prompt": prompt,
"stream": False
}
req_data = {"model": model, "prompt": prompt, "stream": False}
if system:
req_data["system"] = system
if images_b64:
req_data["images"] = images_b64
if format_json:
req_data["format"] = "json"
# Ollama passes configs inside 'options'
if temperature is not None or max_tokens is not None:
req_data["options"] = {}
@@ -290,12 +300,12 @@ def query_llm(
response = requests.post(url, json=req_data, headers=headers, timeout=timeout)
response.raise_for_status()
resp_json = response.json()
# Normalize response payload so callers don't have to distinguish
if is_openai_compat:
# OpenRouter returns choices[0].message.content
content = resp_json.get("choices", [{}])[0].get("message", {}).get("content", "")
usage = resp_json.get("usage", {})
if usage:
cost_str = ""
@@ -306,26 +316,29 @@ def query_llm(
pricing = get_model_pricing(model)
if pricing:
try:
p_cost = float(pricing.get("prompt", 0)) * usage.get('prompt_tokens', 0)
c_cost = float(pricing.get("completion", 0)) * usage.get('completion_tokens', 0)
p_cost = float(pricing.get("prompt", 0)) * usage.get("prompt_tokens", 0)
c_cost = float(pricing.get("completion", 0)) * usage.get("completion_tokens", 0)
calc_cost = p_cost + c_cost
if calc_cost > 0:
cost_str = f" | 💸 Cost: ${calc_cost:.6f}"
except Exception:
pass
p_tokens = usage.get('prompt_tokens', 0)
c_tokens = usage.get('completion_tokens', 0)
t_tokens = usage.get('total_tokens', 0)
p_tokens = usage.get("prompt_tokens", 0)
c_tokens = usage.get("completion_tokens", 0)
t_tokens = usage.get("total_tokens", 0)
# Make it stand out!
logger.info(f"🪙 [LLM Burn] {model} -> In: {p_tokens} | Out: {c_tokens} | Total: {t_tokens}{cost_str}", extra={"color": "\x1b[38;5;208m\x1b[1m"})
logger.info(
f"🪙 [LLM Burn] {model} -> In: {p_tokens} | Out: {c_tokens} | Total: {t_tokens}{cost_str}",
extra={"color": "\x1b[38;5;208m\x1b[1m"},
)
# Validation: if JSON was expected, try to extract it
if format_json:
extracted = extract_json(content)
if not extracted:
raise ValueError(f"OpenRouter returned non-JSON content when JSON was expected: {content[:100]}...")
raise ValueError(f"OpenRouter returned non-JSON content when JSON was expected: {content[:100]}...")
content = extracted
return {"response": content}
@@ -337,31 +350,34 @@ def query_llm(
if not extracted:
# Log more context if JSON extraction fails
logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...")
raise ValueError(f"Ollama returned non-JSON content when JSON was expected.")
raise ValueError("Ollama returned non-JSON content when JSON was expected.")
resp_json["response"] = extracted
return resp_json
except requests.exceptions.ConnectionError:
logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?")
except Exception as e:
logger.error(f"LLM Provider Error with {model}: {e}")
# Prevent infinite fallback loops
if getattr(query_llm, "_is_fallback", False):
return None
# Decide on fallback model/url
f_model = fallback_model
f_url = fallback_url
# Read fallback config from args if available
if not f_model or not f_url:
from GramAddict.core.config import Config
try:
args = Config().args
f_model = f_model or getattr(args, "ai_fallback_model", None)
f_url = f_url or getattr(args, "ai_fallback_url", None)
except Exception:
pass
# Last resort defaults
if not f_model or not f_url:
if is_openai_compat:
@@ -388,12 +404,13 @@ def query_llm(
format_json=format_json,
timeout=timeout,
temperature=temperature,
max_tokens=max_tokens
max_tokens=max_tokens,
)
finally:
query_llm._is_fallback = False
return None
def query_telepathic_llm(
model: str,
url: str,
@@ -401,7 +418,7 @@ def query_telepathic_llm(
user_prompt: str,
temperature: float = 0.0,
use_local_edge: bool = False,
images_b64: Optional[List[str]] = None
images_b64: Optional[List[str]] = None,
) -> str:
"""
Routes UI Telepathic requests purely based on textual interpretation of the screen's XML nodes.
@@ -415,10 +432,13 @@ def query_telepathic_llm(
if use_local_edge:
is_already_local = "localhost" in url or "127.0.0.1" in url
if is_already_local:
logger.debug(f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing.")
logger.debug(
f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing."
)
else:
logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).")
from GramAddict.core.config import Config
try:
args = Config().args
target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
@@ -426,10 +446,10 @@ def query_telepathic_llm(
except Exception:
target_url = "http://localhost:11434/api/generate"
target_model = "llama3.2:1b"
is_local = "localhost" in target_url or "127.0.0.1" in target_url
calc_timeout = 180 if is_local else 45
ans = query_llm(
url=target_url,
model=target_model,
@@ -439,7 +459,7 @@ def query_telepathic_llm(
format_json=True,
timeout=calc_timeout, # Navigation VLM must fail fast for Cloud, but wait for Local VRAM loads
temperature=temperature,
max_tokens=150 # Hard stop to prevent VLM from endlessly hallucinating UI elements
max_tokens=150, # Hard stop to prevent VLM from endlessly hallucinating UI elements
)
if ans and "response" in ans:
return ans["response"]

1
GramAddict/core/navigation/__init__.py Normal file
View File

@@ -0,0 +1 @@
# Navigation domain package

214
GramAddict/core/navigation/knowledge.py Normal file
View File

@@ -0,0 +1,214 @@
import logging
import time
from typing import List, Optional
from GramAddict.core.perception.screen_identity import ScreenType
from GramAddict.core.qdrant_memory import QdrantBase
logger = logging.getLogger(__name__)
class NavigationKnowledge:
"""
Manages the bot's learned understanding of the Instagram UI.
Discovered dynamically through exploration and success.
"""
def __init__(self, username: str):
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
# In-memory cache for rapidly avoiding traps during exploration
# In-memory cache for rapidly avoiding traps during exploration
self._learned_screen_mappings = {}
self._learned_traps = set()
def wipe(self):
"""Wipe all learned knowledge from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.wipe_collection()
except Exception as e:
logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}")
def update_username(self, username: str):
"""Update username and reconnect DB if needed."""
if self.username != username:
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
def get_requirements(self, goal: str) -> List[ScreenType]:
"""Get required screens for a goal. Returns known requirements or empty list."""
if not self._db or not self._db.is_connected:
return []
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="goal", match=MatchValue(value=goal))]),
limit=1,
)[0]
if results:
screen_name = results[0].payload.get("required_screen")
logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}")
if screen_name:
return [ScreenType[screen_name]]
except Exception as e:
logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}")
return []
def learn_goal_requirement(self, goal: str, screen_type: ScreenType):
"""Learn that achieving 'goal' lands us on 'screen_type'."""
if not self._db or not self._db.is_connected:
logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected")
return
seed = f"req_{goal}"
vec = self._db._get_embedding(f"goal_requirement: {goal}")
payload = {"goal": goal, "required_screen": screen_type.name, "timestamp": time.time()}
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}' → {screen_type.name}")
def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]:
"""Find which action leads to this screen."""
for action, screen in self._learned_screen_mappings.items():
if screen == target_screen:
return action
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[FieldCondition(key="result_screen", match=MatchValue(value=target_screen.name))]
),
limit=1,
)[0]
if results:
return results[0].payload.get("action")
except Exception:
pass
return None
def get_screen_for_action(self, action: str) -> Optional[ScreenType]:
"""Find where this action leads to to avoid looping traps."""
if action in self._learned_screen_mappings:
return self._learned_screen_mappings[action]
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="action", match=MatchValue(value=action))]),
limit=1,
)[0]
if results:
screen_name = results[0].payload.get("result_screen")
if screen_name:
return ScreenType[screen_name]
except Exception:
pass
return None
def learn_screen_mapping(self, action: str, result_screen: ScreenType):
"""Learn that taking 'action' leads to 'result_screen'."""
if not self._db or not self._db.is_connected:
return
seed = f"map_{action}"
vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}")
payload = {"action": action, "result_screen": result_screen.name, "timestamp": time.time()}
self._learned_screen_mappings[action] = result_screen
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}' → {result_screen.name}")
def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]:
"""Find where this tab leads to to avoid looping traps."""
if tab_id in self._learned_screen_mappings:
return self._learned_screen_mappings[tab_id]
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(must=[FieldCondition(key="tab_id", match=MatchValue(value=tab_id))]),
limit=1,
)[0]
if results:
s_name = results[0].payload.get("result_screen")
if s_name:
return ScreenType[s_name]
except Exception:
pass
return None
def learn_trap(self, screen_type: ScreenType, action: str, trap_reason: str = "softlock"):
"""Aversively learn that an action on a screen is dangerous/useless."""
trap_key = f"{screen_type.name}_{action}"
self._learned_traps.add(trap_key)
if not self._db or not self._db.is_connected:
return
seed = f"trap_{trap_key}"
# Aversive vector is completely orthogonal to normal goals to prevent retrieval overlap
vec = self._db._get_embedding(f"trap_avoidance: {trap_key} {trap_reason}")
payload = {
"trap_screen": screen_type.name,
"trap_action": action,
"trap_reason": trap_reason,
"timestamp": time.time(),
}
self._db.upsert_point(seed, payload, vector=vec)
logger.error(f"💀 [Aversive Learning] BURNED action '{action}' on {screen_type.name} due to: {trap_reason}")
def is_trap(self, screen_type: ScreenType, action: str) -> bool:
"""Check if an action on this screen is a known trap."""
trap_key = f"{screen_type.name}_{action}"
if trap_key in self._learned_traps:
return True
if not self._db or not self._db.is_connected:
return False
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(key="trap_screen", match=MatchValue(value=screen_type.name)),
FieldCondition(key="trap_action", match=MatchValue(value=action)),
]
),
limit=1,
)[0]
if results:
self._learned_traps.add(trap_key)
return True
except Exception:
pass
return False

117
GramAddict/core/navigation/path_memory.py Normal file
View File

@@ -0,0 +1,117 @@
import logging
import time
from typing import Dict, List, Optional
from GramAddict.core.qdrant_memory import QdrantBase
logger = logging.getLogger(__name__)
class PathMemory:
"""
Qdrant-backed memory for successful navigation paths.
Stores: goal → [step1, step2, ...] → success
Enables instant recall for known goals.
"""
def __init__(self, username: str = ""):
self.username = username
try:
suffix = f"_{username}" if username else ""
self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768)
except Exception:
self._db = None
def wipe(self):
"""Wipe all learned navigation paths from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.wipe_collection()
except Exception as e:
logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}")
def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]:
"""
Recall a previously successful path for this goal from this screen type.
Returns list of steps or None.
"""
if not self._db or not self._db.is_connected:
return None
query = f"goal: {goal} | from: {current_screen_type}"
vec = self._db._get_embedding(query)
if not vec:
return None
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
results = self._db.client.query_points(
collection_name=self._db.collection_name,
query=vec,
query_filter=Filter(
must=[FieldCondition(key="start_screen", match=MatchValue(value=current_screen_type))]
),
limit=3,
score_threshold=0.85,
).points
for r in results:
p = r.payload
if p.get("success") and p.get("steps"):
logger.info(
f"🧠 [GOAP Recall] Found path for '{goal}': "
f"{len(p['steps'])} steps (confidence: {p.get('confidence', 0):.2f})"
)
return p["steps"]
return None
except Exception as e:
logger.debug(f"GOAP recall error: {e}")
return None
def learn_path(self, goal: str, start_screen: str, steps: List[Dict], success: bool):
"""Store a navigation path in Qdrant."""
if not self._db or not self._db.is_connected:
return
query = f"goal: {goal} | from: {start_screen}"
vec = self._db._get_embedding(query)
if not vec:
return
seed = f"{goal}|{start_screen}"
payload = {
"goal": goal,
"start_screen": start_screen,
"steps": steps,
"step_count": len(steps),
"success": success,
"confidence": 0.85 if success else 0.0,
"timestamp": time.time(),
}
outcome = "✅" if success else "❌"
self._db.upsert_point(
seed,
payload,
vector=vec,
log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}",
)
def forget_path(self, goal: str, start_screen: str):
"""Remove a cached path to force re-discovery."""
if not self._db or not self._db.is_connected:
return
seed = f"{goal}|{start_screen}"
try:
from qdrant_client import models
point_id = self._db._get_id(seed)
self._db.client.delete(
collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id])
)
except Exception as e:
logger.debug(f"Failed to forget path: {e}")

171
GramAddict/core/navigation/planner.py Normal file
View File

@@ -0,0 +1,171 @@
import logging
from typing import Any, Dict, List, Optional
from GramAddict.core.navigation.knowledge import NavigationKnowledge
from GramAddict.core.perception.screen_identity import ScreenType
logger = logging.getLogger(__name__)
class GoalPlanner:
"""
Given a goal and current screen state, plans the next action.
Uses Dynamic Discovery to navigate without hardcoded maps.
"""
def __init__(self, username: str):
self.knowledge = NavigationKnowledge(username)
def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]:
"""Plans the NEXT single action to take toward the goal."""
screen_type = screen["screen_type"]
available = screen.get("available_actions", [])
context = screen.get("context", {})
goal_lower = goal.lower()
# ── 1. Check if goal is ALREADY achieved ──
if self._is_goal_achieved(goal_lower, screen_type, context):
logger.info(f"🎯 [GOAP] Goal '{goal}' already achieved on {screen_type.value}!")
return None
# (Phase 5: legacy _plan_goal_action static heuristics purged,
# all intents fall through to VLM-driven Discovery in _plan_navigation)
# ── 3. Am I on the right screen? If not, navigate there ──
selected_tab = screen.get("selected_tab")
nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions)
if nav_action:
return nav_action
# Final fallback: back-track, UNLESS back-tracking is a known trap on this screen!
if not self.knowledge.is_trap(screen_type, "press back"):
return "press back"
# We are trapped! Can't go forward, can't go back!
logger.error(f"💀 [GOAP] Completely trapped on {screen_type.name}. Forcing Instagram restart.")
return "force start instagram"
def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool:
"""Check if the goal is already satisfied. Delegates to ScreenTopology SSOT."""
from GramAddict.core.screen_topology import ScreenTopology
# Interaction goals (context-specific, not navigation)
if "like" in goal and context.get("is_liked") is True:
return True
if "view profile" in goal and screen_type in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE):
return True
# Navigation goals — delegate to SSOT
target = ScreenTopology.goal_to_target_screen(goal)
if target and screen_type == target:
return True
return False
def _plan_navigation(
self,
goal: str,
screen_type: ScreenType,
available: List[str],
selected_tab: Optional[str] = None,
explored_nav_actions: set = None,
) -> Optional[str]:
"""If we're on the wrong screen, figure out how to navigate.
Strategy (priority order):
1. HD Map (ScreenTopology BFS) — deterministic, pre-computed routes
2. Learned Knowledge (Qdrant) — dynamic discovery from past sessions
3. Autonomous Discovery — linguistic matching + VLM intent
"""
from GramAddict.core.screen_topology import ScreenTopology
# 0. Aversive Filter: Remove known traps from available actions
safe_available = []
for action in available:
if not self.knowledge.is_trap(screen_type, action):
safe_available.append(action)
else:
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
available = safe_available
# ── 1. HD Map Routing (Primary Strategy) ──
target_screen = ScreenTopology.goal_to_target_screen(goal)
if target_screen and target_screen != screen_type:
route = ScreenTopology.find_route(screen_type, target_screen)
if route:
next_action, next_screen = route[0]
# Verify action isn't explored/trapped
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
route_desc = " → ".join(s.name for _, s in route)
logger.info(
f"🗺️ [HD Map] Route: {screen_type.name} → {route_desc}. " f"Next action: '{next_action}'"
)
return next_action
else:
logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.")
else:
logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.")
# ── 2. Learned Knowledge (Qdrant) ──
required_screens = self.knowledge.get_requirements(goal)
# ── 3. Autonomous Discovery (Blank Start fallback) ──
if not required_screens:
logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.")
# Return raw intent for TelepathicEngine discovery (VLM)
if explored_nav_actions and goal in explored_nav_actions:
logger.info(
f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking."
)
return None # Don't return goal again — force fallback to press back
else:
return goal
# 4. If we're already on an acceptable screen, no navigation needed
if screen_type in required_screens:
return None
# 5. Find the action we need to take (from learned knowledge or HD map)
for target_screen in required_screens:
# Try HD Map first!
route = ScreenTopology.find_route(screen_type, target_screen)
if route:
next_action, next_screen = route[0]
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
logger.info(f"🧭 [Nav HD Map] Routing to required {target_screen.name} via '{next_action}'")
return next_action
known_action = self.knowledge.get_action_for_screen(target_screen)
if not known_action:
logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...")
screen_friendly_name = target_screen.name.replace("_", " ").lower()
goal_words = [w.rstrip("s") for w in screen_friendly_name.split() if len(w) > 3]
for action in available:
if any(w in action.lower() for w in goal_words):
known_target = self.knowledge.get_screen_for_action(action)
if known_target and known_target != target_screen:
continue
logger.info(
f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'"
)
return action
return f"navigate to {screen_friendly_name}"
else:
if known_action in available:
logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'")
return known_action
# If no targeted navigation works, try going back first
if "press back" in available:
return "press back"
return None

96
GramAddict/core/perception/action_memory.py Normal file
View File

@@ -0,0 +1,96 @@
import logging
from typing import Any, Dict, Optional
from GramAddict.core.perception.spatial_parser import SpatialNode
logger = logging.getLogger(__name__)
class ActionMemory:
"""
Handles the caching, tracking, and negative reinforcement (unlearning) of UI interactions.
Decouples the memory layer from the core parsing engine.
"""
def __init__(self, ui_memory=None):
# We optionally inject UIMemoryDB to decouple tests
if ui_memory is None:
from GramAddict.core.qdrant_memory import UIMemoryDB
self.ui_memory = UIMemoryDB()
else:
self.ui_memory = ui_memory
self._last_click_context: Optional[Dict[str, Any]] = None
def track_click(self, intent: str, node: SpatialNode, xml_context: str = ""):
"""Stores the context of a click before it's actually performed."""
semantic_string = f"text: '{node.text}', desc: '{node.content_desc}', id: '{node.resource_id}'"
self._last_click_context = {
"intent": intent,
"node_dict": node.to_dict(),
"semantic_string": semantic_string,
"xml_context": xml_context,
}
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
def confirm_click(self, intent: str = None):
"""Positive Reinforcement: Confirms the last click was successful."""
ctx = self._last_click_context
if not ctx:
return
if intent and ctx["intent"] != intent:
return
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
# Store or boost in Qdrant
try:
# Check if it exists first
existing = self.ui_memory.retrieve_memory(ctx["intent"], ctx["xml_context"])
if existing:
self.ui_memory.boost_confidence(ctx["intent"], ctx["xml_context"])
else:
self.ui_memory.store_memory(ctx["intent"], ctx["xml_context"], ctx["node_dict"])
except Exception as e:
logger.warning(f"Failed to confirm click in Qdrant: {e}")
self._last_click_context = None
def reject_click(self, intent: str = None):
"""Negative Reinforcement: Penalizes a failed click (Unlearning)."""
ctx = self._last_click_context
if not ctx:
return
if intent and ctx["intent"] != intent:
return
logger.warning(f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.")
try:
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
except Exception as e:
logger.warning(f"Failed to decay confidence in Qdrant: {e}")
self._last_click_context = None
def verify_success(self, intent: str, pre_click_xml: str, post_click_xml: str) -> Optional[bool]:
"""
Structural verification: Did the UI actually change after the click?
"""
# Specific check for explore grid
if "first image in explore grid" in intent or "grid item" in intent:
if "row_feed_photo_imageview" in post_click_xml or "row_feed_button_like" in post_click_xml:
return True
if "explore_action_bar" in post_click_xml and "row_feed_button_like" not in post_click_xml:
return None # Still on grid, inconclusive
if abs(len(pre_click_xml) - len(post_click_xml)) > 50:
logger.debug(f"🧠 [ActionMemory] Structural change detected for '{intent}'. Verification PASS.")
return True
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
return False

63
GramAddict/core/perception/feed_analysis.py
View File

@@ -1,7 +1,7 @@
"""
Perception — Feed Content Analysis.
Structural analysis of the feed: detecting markers, carousels,
Structural analysis of the feed: detecting markers, carousels,
extracting post content. Zero-AI, pure structural parsing.
Extracted from bot_flow.py to enable isolated testing.
@@ -27,14 +27,14 @@ FEED_MARKERS = [
"clips_linear_layout_container",
"zoomable_view_container",
"feed_action_row",
"carousel_viewpager"
"carousel_viewpager",
]
# ── Carousel Detection ──
CAROUSEL_INDICATORS = [
"com.instagram.android:id/carousel_page_indicator",
"com.instagram.android:id/carousel_media_group",
"com.instagram.android:id/carousel_viewpager"
"com.instagram.android:id/carousel_viewpager",
]
@@ -50,26 +50,29 @@ def extract_post_content(context_xml: str) -> dict:
"""
Extracts meaningful content data from the current feed post's XML.
This is the BOT'S EYES — what it actually "sees" about each post.
Returns:
{'username': str, 'description': str, 'caption': str}
"""
result = {"username": "", "description": "", "caption": ""}
try:
from GramAddict.core.telepathic_engine import TelepathicEngine
telepath = TelepathicEngine.get_instance()
# 1. Learn/extract post author dynamically
author_node = telepath.find_best_node(context_xml, "post author username header", min_confidence=0.75)
if author_node and author_node.get("original_attribs", {}).get("text"):
# 🛡️ Anti-Hallucination Guard: The author header is always near the top. Ignore names in the comment section.
if author_node and author_node.get("y", 0) < 1000 and author_node.get("original_attribs", {}).get("text"):
result["username"] = author_node["original_attribs"]["text"].strip()
# 2. Learn/extract post media description dynamically
media_node = telepath.find_best_node(context_xml, "post media content", min_confidence=0.35)
if media_node and media_node.get("original_attribs", {}).get("desc"):
result["description"] = media_node["original_attribs"]["desc"].strip()
# 3. Visible caption text (heuristic fallback if node isn't explicitly found)
# Search all nodes for text that contains the username to find the caption body
root = ET.fromstring(context_xml)
@@ -78,13 +81,51 @@ def extract_post_content(context_xml: str) -> dict:
if result["username"] and len(text) > 20 and result["username"] in text:
result["caption"] = text
break
except Exception as e:
logger.warning(f"Error extracting post content autonomously: {e}")
return result
def _parse_number_from_text(text: str) -> int:
"""Extracts numeric value from strings like '1,234 likes', '1.5M views', 'Gefällt 12.345 Mal'."""
text = text.lower()
# Clean up purely thousands separators but keep decimals
# If there is a 'm' or 'k', a period is usually a decimal (e.g. 1.5m).
# If no 'm' or 'k', a period might be a German thousands separator (12.345).
# We will let the regex handle decimals.
# Remove commas (usually thousands separator in English)
text = text.replace(",", "")
# Find all numbers, potentially with k or m
matches = re.findall(r"(\d+(?:\.\d+)?)\s*([km])?", text)
if not matches:
return 0
best_val = 0
for num_str, multiplier in matches:
val = float(num_str)
if multiplier == "k":
val *= 1000
elif multiplier == "m":
val *= 1000000
else:
# If no multiplier, a period in num_str might be a German thousands separator
if "." in num_str and val < 1000:
# E.g. '12.345' became 12.345. Since no multiplier, it's actually 12345.
# Heuristic: If it has 3 decimal places, it's a thousands separator.
parts = num_str.split(".")
if len(parts[1]) == 3:
val = float(num_str.replace(".", ""))
best_val = max(best_val, int(val))
return best_val
def has_feed_markers(xml_dump: str) -> bool:
"""Quick check: does this XML contain any feed presence markers?"""
return any(marker in xml_dump for marker in FEED_MARKERS)

349
GramAddict/core/perception/screen_identity.py Normal file
View File

@@ -0,0 +1,349 @@
import hashlib
import logging
import re
import xml.etree.ElementTree as ET
from enum import Enum
from typing import Any, Dict
logger = logging.getLogger(__name__)
class ScreenType(Enum):
HOME_FEED = "home_feed"
EXPLORE_GRID = "explore_grid"
REELS_FEED = "reels_feed"
OWN_PROFILE = "own_profile"
OTHER_PROFILE = "other_profile"
POST_DETAIL = "post_detail"
STORY_VIEW = "story_view"
DM_INBOX = "dm_inbox"
DM_THREAD = "dm_thread"
SEARCH_RESULTS = "search_results"
FOLLOW_LIST = "follow_list"
COMMENTS = "comments"
MODAL = "modal"
FOREIGN_APP = "foreign_app"
UNKNOWN = "unknown"
class ScreenIdentity:
"""
Understands what screen the bot is on by analyzing the XML dump.
NO hardcoded states — purely structural analysis.
This is the bot's EYES. It answers: "What do I see right now?"
"""
def __init__(self, bot_username: str):
self.bot_username = bot_username.lower()
try:
from GramAddict.core.qdrant_memory import ScreenMemoryDB
self.screen_memory = ScreenMemoryDB()
except ImportError:
self.screen_memory = None
def identify(self, xml_dump: str) -> Dict[str, Any]:
"""
Analyzes an XML dump and returns a complete screen description.
Returns:
{
'screen_type': ScreenType,
'available_actions': ['tap like button', 'tap explore tab', ...],
'selected_tab': 'feed_tab' | 'search_tab' | ...,
'context': {'username': '...', 'post_count': '...', ...}
}
"""
if not xml_dump or not isinstance(xml_dump, str):
return self._empty_screen()
try:
clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip()
root = ET.fromstring(clean)
except Exception:
return self._empty_screen()
# Extract structural signals
packages = set()
resource_ids = set()
content_descs = []
texts = []
selected_tab = None
clickable_elements = []
app_id = "com.instagram.android"
for elem in root.iter("node"):
pkg = elem.get("package", "")
if pkg:
packages.add(pkg)
rid = elem.get("resource-id", "").strip()
text = elem.get("text", "").strip()
desc = elem.get("content-desc", "").strip()
clickable = elem.get("clickable", "false") == "true"
selected = elem.get("selected", "false") == "true"
bounds = elem.get("bounds", "")
if rid:
# Normalize: "com.instagram.android:id/feed_tab" → "feed_tab"
short_id = rid.split("/")[-1] if "/" in rid else rid
resource_ids.add(short_id)
# Track which tab is selected
if selected and short_id in ("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab"):
selected_tab = short_id
if text:
texts.append(text)
if desc:
content_descs.append(desc)
if clickable and bounds:
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
if match:
left, t, r, b = map(int, match.groups())
cx, cy = (left + r) // 2, (t + b) // 2
clickable_elements.append(
{
"text": text,
"desc": desc,
"id": rid.split("/")[-1] if "/" in rid else rid,
"x": cx,
"y": cy,
"bounds": bounds,
}
)
# ── Foreign app check ──
if app_id not in packages:
return {
"screen_type": ScreenType.FOREIGN_APP,
"available_actions": ["press back", "force start instagram"],
"selected_tab": None,
"context": {"packages": list(packages)},
"signature": self._compute_signature(resource_ids, content_descs, texts),
}
desc_lower = " ".join(content_descs).lower()
text_lower = " ".join(texts).lower()
ids_str = " ".join(resource_ids).lower()
signature = self._compute_signature(resource_ids, content_descs, texts)
# ── Identify screen type from structural signals ──
screen_type = self._classify_screen(
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature
)
# ── Extract available actions from clickable elements ──
available_actions = self._extract_available_actions(
clickable_elements, resource_ids, content_descs, texts, screen_type
)
# ── Extract context ──
context = self._extract_context(content_descs, texts, resource_ids, screen_type)
return {
"screen_type": screen_type,
"available_actions": available_actions,
"selected_tab": selected_tab,
"context": context,
"signature": signature,
}
def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None):
"""Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
# Priority 0: Content-creation overlays that block ALL navigation.
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
if any(marker in ids_str for marker in creation_flow_markers):
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
return ScreenType.MODAL
# Priority 1: Check Qdrant Semantic Cache
if signature and self.screen_memory and self.screen_memory.is_connected:
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
if cached_type_str:
try:
return ScreenType[cached_type_str]
except KeyError:
pass
# Priority 2: Structural Heuristics (Instant, for core tabs)
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
return ScreenType.FOLLOW_LIST
if "profile_header_container" in ids:
return ScreenType.OTHER_PROFILE
# Reels structural markers — present even when Instagram hides the tab bar
# in full-screen Reels viewing. Without this, selected_tab=None → UNKNOWN.
REELS_MARKERS = ("clips_viewer_container", "root_clips_layout", "clips_linear_layout_container")
if any(marker in ids for marker in REELS_MARKERS):
return ScreenType.REELS_FEED
# DM thread detection — structural markers present inside DM conversations
if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids:
return ScreenType.DM_THREAD
if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab:
return ScreenType.POST_DETAIL
if selected_tab == "feed_tab":
return ScreenType.HOME_FEED
if selected_tab == "clips_tab":
return ScreenType.REELS_FEED
if selected_tab == "search_tab":
return ScreenType.EXPLORE_GRID
if selected_tab == "profile_tab":
return ScreenType.OWN_PROFILE
if selected_tab == "direct_tab":
return ScreenType.DM_INBOX
if "message_input" in ids:
return ScreenType.DM_INBOX # Fallback for DM thread as inbox
# Priority 3: Semantic VLM Classification Fallback
from GramAddict.core.config import Config
from GramAddict.core.llm_provider import query_llm
cfg = Config()
url = (
getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat")
if hasattr(cfg, "args")
else "http://localhost:11434/api/chat"
)
model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3"
layout_context = (
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
)
prompt = (
f"Identify the Instagram screen layout type based on these DOM structural signals.\n"
f"Valid types: {[t.name for t in ScreenType]}\n"
f"Context:\n{layout_context}\n"
f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches."
)
try:
response = query_llm(
url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False
)
if response and isinstance(response, str):
result = response.strip().upper()
elif response and isinstance(response, dict) and "response" in response:
result = response["response"].strip().upper()
else:
return ScreenType.UNKNOWN
for t in ScreenType:
if t.name in result:
if signature and self.screen_memory:
self.screen_memory.store_screen(signature, t.name)
return t
except Exception as e:
import logging
logging.getLogger(__name__).debug(f"LLM Classification failed: {e}")
return ScreenType.UNKNOWN
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type):
"""Discover what actions are possible on this screen."""
actions = []
# Navigation tabs (always available when visible)
tab_map = {
"feed_tab": "tap home tab",
"search_tab": "tap explore tab",
"clips_tab": "tap reels tab",
"profile_tab": "tap profile tab",
"direct_tab": "tap messages tab",
}
for tab_id, action in tab_map.items():
if tab_id in resource_ids:
actions.append(action)
# Screen-specific actions
desc_lower = " ".join(content_descs).lower()
text_lower = " ".join(texts).lower()
if "like" in desc_lower:
actions.append("tap like button")
if "comment" in desc_lower:
actions.append("tap comment button")
if "share" in desc_lower:
actions.append("tap share button")
if "save" in desc_lower or "bookmark" in desc_lower:
actions.append("tap save button")
if "back" in desc_lower:
actions.append("tap back button")
if any("follow" in e.get("text", "").lower() for e in clickable_elements):
actions.append("tap follow button")
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
if "message" in desc_lower or "nachricht" in desc_lower:
actions.append("tap message button")
if (
"following" in desc_lower
or "abonniert" in desc_lower
or "following" in text_lower
or "profile_header_following" in " ".join(resource_ids).lower()
):
actions.append("tap following list")
# Grid items
if screen_type == ScreenType.EXPLORE_GRID:
actions.append("tap first grid item")
# Scroll
actions.append("scroll down")
actions.append("press back")
return list(set(actions)) # Deduplicate
def _extract_context(self, content_descs, texts, resource_ids, screen_type):
"""Extract meaningful context from the screen."""
context = {}
desc_text = " ".join(content_descs)
# Username on profile
username_match = re.search(r"(\w+)'s (?:profile|story|unseen story)", desc_text)
if username_match:
context["username"] = username_match.group(1)
# Post/follower counts
for d in content_descs:
m = re.match(r"([\d,.]+K?M?)(\s*)(posts?|followers?|following)", d, re.IGNORECASE)
if m:
context[m.group(3).lower()] = m.group(1)
# Like state
for d in content_descs:
if d.lower() == "liked":
context["is_liked"] = True
elif d.lower() == "like":
context["is_liked"] = False
return context
def _compute_signature(self, resource_ids, content_descs, texts):
"""Compute a stable hash for this screen state (for Qdrant lookup)."""
# Use sorted IDs + key content for stability
sig_parts = sorted(resource_ids)[:20]
sig_parts.extend(sorted(set(d.lower()[:30] for d in content_descs if len(d) > 2))[:10])
sig = "|".join(sig_parts)
return hashlib.sha256(sig.encode()).hexdigest()[:24]
def _empty_screen(self):
return {
"screen_type": ScreenType.FOREIGN_APP,
"available_actions": ["press back", "force start instagram"],
"selected_tab": None,
"context": {},
"signature": "empty",
}

144
GramAddict/core/perception/semantic_evaluator.py Normal file
View File

@@ -0,0 +1,144 @@
import json
import logging
import re
from typing import List, Optional
from GramAddict.core.llm_provider import query_telepathic_llm
from GramAddict.core.perception.spatial_parser import SpatialNode
logger = logging.getLogger(__name__)
class SemanticEvaluator:
"""
Handles LLM/VLM interaction for high-level semantic analysis of the UI.
Delegates vision processing and prompt engineering out of the core routing engine.
"""
def __init__(self):
from GramAddict.core.config import Config
try:
self.args = Config().args
except Exception:
self.args = None
def _query_vlm(self, prompt: str, screenshot_b64: str) -> Optional[str]:
if not self.args:
logger.warning("👁️ [Vision Core] No config available. Cannot query VLM.")
return None
model = getattr(self.args, "ai_telepathic_model", "llama3.2-vision")
url = getattr(self.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
try:
res = query_telepathic_llm(
model=model,
url=url,
system_prompt="You are an expert Instagram assistant.",
user_prompt=prompt,
images_b64=[screenshot_b64],
)
return res
except Exception as e:
logger.error(f"👁️ [Vision Core] LLM query failed: {e}")
return None
def evaluate_grid_visuals(
self, device, persona_interests: list[str], grid_nodes: List[SpatialNode]
) -> Optional[SpatialNode]:
"""
Takes the spatial grid nodes and asks the VLM which one best matches the persona.
"""
logger.info(f"👁️ [Vision Core] Analyzing grid aesthetics against niche interests: {persona_interests}...")
if not grid_nodes:
return None
# Take a screenshot
try:
screenshot_b64 = device.get_screenshot_b64()
except Exception as e:
logger.error(f"👁️ [Vision Core] Failed to capture screenshot: {e}")
return None
simplified_nodes = []
for i, node in enumerate(grid_nodes[:9]): # Limit to 9 to save tokens
simplified_nodes.append({"index": i, "bounds": node.bounds})
prompt = f"""
You are a highly perceptive Instagram user with the following interests: {', '.join(persona_interests)}.
Look at the provided screenshot of the Instagram Explore/Profile grid.
Below are the bounding boxes for the top grid posts currently visible.
{simplified_nodes}
Your task:
1. Identify which of these posts visually aligns BEST with your interests.
2. Reply ONLY in JSON format: {{"index": <int>}}
3. If absolutely none of them are relevant, reply with {{"index": -1}}.
"""
try:
response = self._query_vlm(prompt, screenshot_b64)
if not response:
return None
try:
data = json.loads(response)
idx = data.get("index", -1)
if idx == -1:
logger.info("👁️ [Vision Core] VLM rejected all grid items. Will scroll down.")
return None
if 0 <= idx < len(grid_nodes):
logger.info(f"👁️ [Vision Core] VLM selected grid item index [{idx}] as the best match.")
return grid_nodes[idx]
except json.JSONDecodeError:
# Fallback to fuzzy
clean_res = response.strip().upper()
match = re.search(r"\d+", clean_res)
if match:
idx = int(match.group())
if 0 <= idx < len(grid_nodes):
logger.info(f"👁️ [Vision Core] VLM selected grid item index [{idx}] as the best match.")
return grid_nodes[idx]
except Exception as e:
logger.warning(f"👁️ [Vision Core] Exception during grid evaluation: {e}")
return None
def evaluate_post_vibe(self, device, persona_interests: list[str]) -> Optional[dict]:
"""Evaluates whether the currently viewed post aligns with persona interests."""
logger.info(f"👁️ [Vision Core] Evaluating post vibe against: {persona_interests}")
try:
screenshot_b64 = device.get_screenshot_b64()
prompt = f"""
You are a user with the following interests: {', '.join(persona_interests)}.
You are looking at an Instagram post.
Evaluate if this post is highly relevant to your interests and if you should like/comment on it.
Reply ONLY in valid JSON format:
{{
"should_like": true/false,
"should_comment": true/false,
"reasoning": "brief explanation"
}}
"""
response = self._query_vlm(prompt, screenshot_b64)
if response:
if "```json" in response:
json_str = response.split("```json")[1].split("```")[0].strip()
else:
json_str = response.strip()
return json.loads(json_str)
except Exception as e:
logger.warning(f"Failed to evaluate post vibe: {e}")
return None
def evaluate_profile_vibe(self, device, persona_interests: list[str]) -> Optional[dict]:
"""Evaluates if a profile is worth following."""
pass
def classify_screen_content(self, xml_hierarchy: str, target_class: str) -> Optional[str]:
pass

194
GramAddict/core/perception/spatial_parser.py Normal file
View File

@@ -0,0 +1,194 @@
import re
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
@dataclass
class SpatialNode:
"""A single node in the Spatial Graph, representing a UI element and its geometry."""
bounds: Tuple[int, int, int, int] # (x1, y1, x2, y2)
node_id: str = ""
class_name: str = ""
text: str = ""
content_desc: str = ""
resource_id: str = ""
clickable: bool = False
scrollable: bool = False
# Spatial Properties
children: List["SpatialNode"] = field(default_factory=list)
parent: Optional["SpatialNode"] = None
@property
def x1(self) -> int:
return self.bounds[0]
@property
def y1(self) -> int:
return self.bounds[1]
@property
def x2(self) -> int:
return self.bounds[2]
@property
def y2(self) -> int:
return self.bounds[3]
@property
def width(self) -> int:
return self.x2 - self.x1
@property
def height(self) -> int:
return self.y2 - self.y1
@property
def center_x(self) -> int:
return self.x1 + (self.width // 2)
@property
def center_y(self) -> int:
return self.y1 + (self.height // 2)
@property
def area(self) -> int:
return self.width * self.height
def contains(self, other: "SpatialNode") -> bool:
"""Returns True if this node completely encompasses the other node geometrically."""
return self.x1 <= other.x1 and self.y1 <= other.y1 and self.x2 >= other.x2 and self.y2 >= other.y2
def intersects(self, other: "SpatialNode") -> bool:
"""Returns True if this node's bounding box overlaps with the other's bounding box."""
if self.x1 >= other.x2 or other.x1 >= self.x2:
return False
if self.y1 >= other.y2 or other.y1 >= self.y2:
return False
return True
def to_dict(self) -> Dict[str, Any]:
return {
"id": self.node_id,
"class": self.class_name,
"text": self.text,
"content_desc": self.content_desc,
"resource_id": self.resource_id,
"bounds": self.bounds,
"clickable": self.clickable,
"scrollable": self.scrollable,
"center": (self.center_x, self.center_y),
}
class SpatialParser:
"""
Parses Android UI XML into a structured 2D Spatial Tree.
Calculates parent-child relationships structurally, not just based on XML nesting.
"""
def __init__(self):
self._node_counter = 0
def parse(self, xml_string: str) -> Optional[SpatialNode]:
"""Parses the raw XML dump into a Spatial Graph."""
try:
clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_string).strip()
if not clean_xml:
return None
root_elem = ET.fromstring(clean_xml)
# 1. First Pass: Create flat list of spatial nodes
all_nodes = []
self._flatten_xml(root_elem, all_nodes)
if not all_nodes:
return None
# 2. Second Pass: Reconstruct tree based on strict spatial containment
# Sort nodes by area descending (largest first)
all_nodes.sort(key=lambda n: n.area, reverse=True)
root_node = all_nodes[0]
for i in range(1, len(all_nodes)):
child = all_nodes[i]
# Find the smallest node that contains this child
# Since we sorted by area descending, we search backwards to find the tightest fit
parent_found = False
for j in range(i - 1, -1, -1):
potential_parent = all_nodes[j]
if potential_parent.contains(child):
potential_parent.children.append(child)
child.parent = potential_parent
parent_found = True
break
# Fallback to root if no parent found (floating node)
if not parent_found and child != root_node:
root_node.children.append(child)
child.parent = root_node
return root_node
except ET.ParseError:
return None
def _flatten_xml(self, element: ET.Element, nodes_list: List[SpatialNode]):
"""Recursively traverses the XML and creates a flat list of SpatialNodes."""
attrib = element.attrib
bounds_str = attrib.get("bounds", "")
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
if match:
left, top, right, bottom = map(int, match.groups())
# Filter zero-area nodes early
if right > left and bottom > top:
self._node_counter += 1
node = SpatialNode(
node_id=f"n_{self._node_counter}",
class_name=attrib.get("class", ""),
text=attrib.get("text", "").strip(),
content_desc=attrib.get("content-desc", "").strip(),
resource_id=attrib.get("resource-id", "").strip(),
bounds=(left, top, right, bottom),
clickable=attrib.get("clickable", "false") == "true",
scrollable=attrib.get("scrollable", "false") == "true",
)
nodes_list.append(node)
for child in element:
self._flatten_xml(child, nodes_list)
def get_all_nodes(self, root: SpatialNode) -> List[SpatialNode]:
"""Flattens the Spatial Tree into a list for easy filtering."""
result = [root]
for child in root.children:
result.extend(self.get_all_nodes(child))
return result
def get_clickable_nodes(self, root: SpatialNode) -> List[SpatialNode]:
"""Returns all nodes that are clickable or have strong semantic meaning."""
all_nodes = self.get_all_nodes(root)
clickables = []
for n in all_nodes:
has_semantic = bool(n.text or n.content_desc)
semantic_res = n.resource_id and any(
x in n.resource_id.lower() for x in ["button", "tab", "icon", "action", "menu"]
)
if n.clickable or n.scrollable or semantic_res or (has_semantic and n.area < 500000 and n.area > 0):
# Filter out pure massive containers (like whole screen) if they aren't explicitly clickable
if not n.clickable and not n.scrollable and n.area > 2000000:
continue
# Also exclude if it's just a ViewGroup with a description but no action
if not n.clickable and n.class_name == "android.view.ViewGroup":
continue
clickables.append(n)
return clickables

494
GramAddict/core/qdrant_memory.py
View File

File diff suppressed because it is too large Load Diff

17
GramAddict/core/screen_topology.py
View File

@@ -8,8 +8,8 @@ This is the bot's GPS: it knows HOW to get from screen A to screen B
before the bot starts moving. The GOAP planner consults this map
as its primary routing strategy.
"""
from collections import deque
from enum import Enum
from typing import Dict, List, Optional, Tuple
from GramAddict.core.goap import ScreenType
@@ -38,6 +38,7 @@ class ScreenTopology:
"tap home tab": ScreenType.HOME_FEED,
"tap profile tab": ScreenType.OWN_PROFILE,
"tap reels tab": ScreenType.REELS_FEED,
"view a post": ScreenType.POST_DETAIL,
},
ScreenType.REELS_FEED: {
"tap home tab": ScreenType.HOME_FEED,
@@ -78,12 +79,16 @@ class ScreenTopology:
"open messages": ScreenType.DM_INBOX,
"open following list": ScreenType.FOLLOW_LIST,
"open followers list": ScreenType.FOLLOW_LIST,
"view a post": ScreenType.POST_DETAIL,
"open post": ScreenType.POST_DETAIL,
"open post author profile": ScreenType.OTHER_PROFILE,
"view the user profile": ScreenType.OTHER_PROFILE,
"view user profile": ScreenType.OTHER_PROFILE,
"open user profile": ScreenType.OTHER_PROFILE,
}
@classmethod
def find_route(
cls, from_screen: ScreenType, to_screen: ScreenType
) -> Optional[List[Tuple[str, ScreenType]]]:
def find_route(cls, from_screen: ScreenType, to_screen: ScreenType) -> Optional[List[Tuple[str, ScreenType]]]:
"""
BFS shortest path from from_screen to to_screen.
@@ -171,9 +176,7 @@ class ScreenTopology:
return f"navigate to {screen_name}"
@classmethod
def expected_screen_for_action(
cls, action: str, from_screen: ScreenType
) -> Optional[ScreenType]:
def expected_screen_for_action(cls, action: str, from_screen: ScreenType) -> Optional[ScreenType]:
"""What screen should we land on after this action from this screen?
Used by _execute_action to validate INTERMEDIATE navigation steps.

85
GramAddict/core/session_state.py
View File

@@ -80,64 +80,40 @@ class SessionState:
self,
):
"""set the limits for current session"""
self.args.current_likes_limit = get_value(
getattr(self.args, "total_likes_limit", 300), None, 300
)
self.args.current_follow_limit = get_value(
getattr(self.args, "total_follows_limit", 50), None, 50
)
self.args.current_unfollow_limit = get_value(
getattr(self.args, "total_unfollows_limit", 50), None, 50
)
self.args.current_comments_limit = get_value(
getattr(self.args, "total_comments_limit", 10), None, 10
)
self.args.current_likes_limit = get_value(getattr(self.args, "total_likes_limit", 300), None, 300)
self.args.current_follow_limit = get_value(getattr(self.args, "total_follows_limit", 50), None, 50)
self.args.current_unfollow_limit = get_value(getattr(self.args, "total_unfollows_limit", 50), None, 50)
self.args.current_comments_limit = get_value(getattr(self.args, "total_comments_limit", 10), None, 10)
self.args.current_pm_limit = get_value(getattr(self.args, "total_pm_limit", 10), None, 10)
self.args.current_watch_limit = get_value(
getattr(self.args, "total_watches_limit", 50), None, 50
)
self.args.current_watch_limit = get_value(getattr(self.args, "total_watches_limit", 50), None, 50)
self.args.current_success_limit = get_value(
getattr(self.args, "total_successful_interactions_limit", 100), None, 100
)
self.args.current_total_limit = get_value(
getattr(self.args, "total_interactions_limit", 1000), None, 1000
)
self.args.current_scraped_limit = get_value(
getattr(self.args, "total_scraped_limit", 200), None, 200
)
self.args.current_crashes_limit = get_value(
getattr(self.args, "total_crashes_limit", 5), None, 5
)
self.args.current_total_limit = get_value(getattr(self.args, "total_interactions_limit", 1000), None, 1000)
self.args.current_scraped_limit = get_value(getattr(self.args, "total_scraped_limit", 200), None, 200)
self.args.current_crashes_limit = get_value(getattr(self.args, "total_crashes_limit", 5), None, 5)
def check_limit(self, limit_type=None, output=False):
"""Returns True if limit reached - else False"""
limit_type = SessionState.Limit.ALL if limit_type is None else limit_type
# check limits
total_likes = self.totalLikes >= int(self.args.current_likes_limit)
total_followed = sum(self.totalFollowed.values()) >= int(
self.args.current_follow_limit
)
total_followed = sum(self.totalFollowed.values()) >= int(self.args.current_follow_limit)
total_unfollowed = self.totalUnfollowed >= int(self.args.current_unfollow_limit)
total_comments = self.totalComments >= int(self.args.current_comments_limit)
total_pm = self.totalPm >= int(self.args.current_pm_limit)
total_watched = self.totalWatched >= int(self.args.current_watch_limit)
total_successful = sum(self.successfulInteractions.values()) >= int(
self.args.current_success_limit
)
total_interactions = sum(self.totalInteractions.values()) >= int(
self.args.current_total_limit
)
total_successful = sum(self.successfulInteractions.values()) >= int(self.args.current_success_limit)
total_interactions = sum(self.totalInteractions.values()) >= int(self.args.current_total_limit)
total_scraped = sum(self.totalScraped.values()) >= int(
self.args.current_scraped_limit
)
total_scraped = sum(self.totalScraped.values()) >= int(self.args.current_scraped_limit)
total_crashes = self.totalCrashes >= int(self.args.current_crashes_limit)
session_info = [
"Checking session limits:",
f"- Total Likes:\t\t\t\t{'Limit Reached' if total_likes else 'OK'} ({self.totalLikes}/{self.args.current_likes_limit})",
f"- Total Comments:\t\t\t\t{'Limit Reached' if total_comments else 'OK'} ({self.totalComments}/{self.args.current_comments_limit})",
f"- Session Likes Given:\t\t{'Limit Reached' if total_likes else 'OK'} ({self.totalLikes}/{self.args.current_likes_limit})",
f"- Session Comments Given:\t{'Limit Reached' if total_comments else 'OK'} ({self.totalComments}/{self.args.current_comments_limit})",
f"- Total PM:\t\t\t\t\t{'Limit Reached' if total_pm else 'OK'} ({self.totalPm}/{self.args.current_pm_limit})",
f"- Total Followed:\t\t\t\t{'Limit Reached' if total_followed else 'OK'} ({sum(self.totalFollowed.values())}/{self.args.current_follow_limit})",
f"- Total Unfollowed:\t\t\t\t{'Limit Reached' if total_unfollowed else 'OK'} ({self.totalUnfollowed}/{self.args.current_unfollow_limit})",
@@ -154,11 +130,16 @@ class SessionState:
logger.info(line)
return (
total_likes and getattr(self.args, "end_if_likes_limit_reached", False)
or total_followed and getattr(self.args, "end_if_follows_limit_reached", False)
or total_watched and getattr(self.args, "end_if_watches_limit_reached", False)
or total_comments and getattr(self.args, "end_if_comments_limit_reached", False)
or total_pm and getattr(self.args, "end_if_pm_limit_reached", False),
total_likes
and getattr(self.args, "end_if_likes_limit_reached", False)
or total_followed
and getattr(self.args, "end_if_follows_limit_reached", False)
or total_watched
and getattr(self.args, "end_if_watches_limit_reached", False)
or total_comments
and getattr(self.args, "end_if_comments_limit_reached", False)
or total_pm
and getattr(self.args, "end_if_pm_limit_reached", False),
total_unfollowed,
total_interactions or total_successful or total_scraped,
)
@@ -247,20 +228,20 @@ class SessionState:
delta = timedelta(seconds=delta_sec)
if not working_hours:
return True, 0
for n in working_hours:
today = current_time.strftime("%Y-%m-%d")
# 100% Autonomous: Hybrid Time Format Support (Legacy . vs Modern :)
h_start = n.split('-')[0].replace(":", ".")
h_end = n.split('-')[1].replace(":", ".")
h_start = n.split("-")[0].replace(":", ".")
h_end = n.split("-")[1].replace(":", ".")
inf_value = f"{h_start} {today}"
inf = datetime.strptime(inf_value, "%H.%M %Y-%m-%d") + delta
sup_value = f"{h_end} {today}"
sup = datetime.strptime(sup_value, "%H.%M %Y-%m-%d") + delta
if sup - inf + timedelta(minutes=1) == timedelta(
days=1
) or sup - inf + timedelta(minutes=1) == timedelta(days=0):
if sup - inf + timedelta(minutes=1) == timedelta(days=1) or sup - inf + timedelta(minutes=1) == timedelta(
days=0
):
logger.debug("Whole day mode.")
return True, 0
if time_in_range(inf.time(), sup.time(), current_time.time()):
@@ -300,9 +281,7 @@ class SessionStateEncoder(JSONEncoder):
return {
"id": session_state.id,
"total_interactions": sum(session_state.totalInteractions.values()),
"successful_interactions": sum(
session_state.successfulInteractions.values()
),
"successful_interactions": sum(session_state.successfulInteractions.values()),
"total_followed": sum(session_state.totalFollowed.values()),
"total_likes": session_state.totalLikes,
"total_comments": session_state.totalComments,

276
GramAddict/core/situational_awareness.py
View File

@@ -10,13 +10,13 @@ After initial learning, 95%+ of situations are handled from memory
alone with ZERO LLM calls. This is "Tesla fleet learning" for bots.
"""
import logging
import hashlib
import time
import logging
import re
import time
import xml.etree.ElementTree as ET
from typing import Optional, Dict, Any
from enum import Enum
from typing import Dict, Optional
from GramAddict.core.utils import random_sleep
@@ -34,6 +34,7 @@ class SituationType(Enum):
class EscapeAction:
"""Represents a planned escape action."""
def __init__(self, action_type: str, x: int = 0, y: int = 0, reason: str = "", resource_id: str = ""):
self.action_type = action_type # 'click', 'back', 'app_start', 'home_then_app'
self.x = x
@@ -42,11 +43,19 @@ class EscapeAction:
self.resource_id = resource_id
def to_dict(self) -> dict:
return {"action_type": self.action_type, "x": self.x, "y": self.y, "reason": self.reason, "resource_id": self.resource_id}
return {
"action_type": self.action_type,
"x": self.x,
"y": self.y,
"reason": self.reason,
"resource_id": self.resource_id,
}
@classmethod
def from_dict(cls, d: dict) -> "EscapeAction":
return cls(d.get("action_type", "back"), d.get("x", 0), d.get("y", 0), d.get("reason", ""), d.get("resource_id", ""))
return cls(
d.get("action_type", "back"), d.get("x", 0), d.get("y", 0), d.get("reason", ""), d.get("resource_id", "")
)
class SituationEpisodeDB:
@@ -56,8 +65,10 @@ class SituationEpisodeDB:
Enables instant recall for known situations (0 LLM calls).
Stores BOTH positive and negative episodes for full learning.
"""
def __init__(self):
from GramAddict.core.qdrant_memory import QdrantBase
self._db = QdrantBase("sae_episodes_v1", vector_size=768)
def recall(self, situation_signature: str) -> Optional[Dict]:
@@ -126,9 +137,31 @@ class SituationEpisodeDB:
if not vec:
return
# Unique key: situation + action type + success flag
seed = f"{situation_signature}|{action.action_type}|{action.x},{action.y}|{success}"
confidence = 0.8 if success else 0.0
# Unique key: situation + action type (ignoring success flag for the seed so we update the same entry)
seed = f"{situation_signature}|{action.action_type}|{action.x},{action.y}"
point_id = self._db.generate_uuid(seed)
current_conf = 0.0
has_existing = False
try:
points = self._db.client.retrieve(
collection_name=self._db.collection_name, ids=[point_id], with_payload=True, with_vectors=False
)
if points:
has_existing = True
current_conf = points[0].payload.get("confidence", 0.0)
except Exception:
pass
if success:
confidence = min(1.0, current_conf + 0.5) if has_existing else 0.8
else:
confidence = current_conf - 0.5 if has_existing else -0.5
if confidence < 0.1 and not success:
self._db.client.delete(collection_name=self._db.collection_name, points_selector=[point_id])
logger.info("🗑️ [SAE Learn] Action decayed below threshold. Deleted from memory.")
return
payload = {
"situation": situation_signature[:500],
@@ -141,8 +174,10 @@ class SituationEpisodeDB:
outcome = "✅ SUCCESS" if success else "❌ FAILURE"
self._db.upsert_point(
seed, payload, vector=vec,
log_success=f"🧠 [SAE Learn] {outcome}: '{action.reason}' → Stored for future recall"
seed,
payload,
vector=vec,
log_success=f"🧠 [SAE Learn] {outcome}: '{action.reason}' → Stored for future recall",
)
def boost(self, situation_signature: str, action: EscapeAction):
@@ -193,7 +228,7 @@ class SituationalAwarenessEngine:
try:
# Remove XML declaration
clean = re.sub(r'<\?xml.*?\?>', '', xml_dump).strip()
clean = re.sub(r"<\?xml.*?\?>", "", xml_dump).strip()
root = ET.fromstring(clean)
except Exception:
# If XML is broken, extract what we can with regex
@@ -205,17 +240,17 @@ class SituationalAwarenessEngine:
packages = set()
elements = []
for elem in root.iter('node'):
for elem in root.iter("node"):
a = elem.attrib
pkg = a.get('package', '')
pkg = a.get("package", "")
if pkg:
packages.add(pkg)
rid = a.get('resource-id', '').strip()
text = a.get('text', '').strip()
desc = a.get('content-desc', '').strip()
bounds = a.get('bounds', '')
clickable = a.get('clickable', 'false')
rid = a.get("resource-id", "").strip()
text = a.get("text", "").strip()
desc = a.get("content-desc", "").strip()
bounds = a.get("bounds", "")
clickable = a.get("clickable", "false")
# Only keep nodes with meaningful content
if not rid and not text and not desc:
@@ -225,10 +260,14 @@ class SituationalAwarenessEngine:
if rid:
parts.append(f"id={rid.split('/')[-1]}")
if text:
parts.append(f"text='{text[:60]}'")
if len(text) > 20:
text = text[:10] + "..." + text[-10:]
parts.append(f"text='{text}'")
if desc:
parts.append(f"desc='{desc[:60]}'")
if clickable == 'true':
if len(desc) > 20:
desc = desc[:10] + "..." + desc[-10:]
parts.append(f"desc='{desc}'")
if clickable == "true":
parts.append("CLICKABLE")
if bounds:
parts.append(f"bounds={bounds}")
@@ -236,14 +275,14 @@ class SituationalAwarenessEngine:
elements.append(" | ".join(parts))
sig = f"PACKAGES: {', '.join(sorted(packages))}\n"
sig += "\n".join(elements[:50]) # Cap at 50 elements
sig += "\n".join(elements[-50:]) # Keep the last 50 elements (highest Z-index/foreground)
return sig[:3000]
def _compute_situation_hash(self, compressed: str) -> str:
"""Deterministic hash for situation dedup."""
# Remove volatile parts (timestamps, counters) but keep structural identity
stable = re.sub(r'\d{2}:\d{2}', 'HH:MM', compressed)
stable = re.sub(r'Battery \d+ per cent', 'Battery NN per cent', stable)
stable = re.sub(r"\d{2}:\d{2}", "HH:MM", compressed)
stable = re.sub(r"Battery \d+ per cent", "Battery NN per cent", stable)
return hashlib.sha256(stable.encode()).hexdigest()[:32]
def perceive(self, xml_dump: str) -> SituationType:
@@ -257,12 +296,17 @@ class SituationalAwarenessEngine:
xml_lower = xml_dump.lower()
blocked_markers = [
"try again later", "action blocked", "restrict certain activity",
"help us confirm you own", "confirm it's you",
"später erneut versuchen", "bestätige, dass du es bist",
"handlung blockiert", "eingeschränkt",
"try again later",
"action blocked",
"restrict certain activity",
"help us confirm you own",
"confirm it's you",
"später erneut versuchen",
"bestätige, dass du es bist",
"handlung blockiert",
"eingeschränkt",
]
# Guard: Check if the text matches are relatively isolated (e.g. short strings).
# If the string is buried inside a 200-character caption, it's a false positive.
# We can regex match text="..." attributes that are less than 60 characters total,
@@ -274,18 +318,18 @@ class SituationalAwarenessEngine:
return SituationType.DANGER_ACTION_BLOCKED
# ── Hardware Guard: Screen Off / Locked ──
if not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True):
if not getattr(self.device.deviceV2, "info", {}).get("screenOn", True):
logger.info("📱 [SAE Perceive] Screen is physically OFF.")
return SituationType.OBSTACLE_LOCKED_SCREEN
# ── System Dialog / Permission Detect (Fast Path) ──
packages = set(re.findall(r'package=["\']([^"\']+)["\']', xml_dump))
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
app_id = getattr(self.device, "app_id", "com.instagram.android")
system_dialog_pkgs = {
'com.google.android.permissioncontroller',
'com.android.permissioncontroller',
'com.samsung.android.permissioncontroller'
"com.google.android.permissioncontroller",
"com.android.permissioncontroller",
"com.samsung.android.permissioncontroller",
}
if any(pkg in system_dialog_pkgs for pkg in packages):
logger.info("📱 [SAE Perceive] System permission dialog explicitly detected.")
@@ -294,7 +338,7 @@ class SituationalAwarenessEngine:
# ── Foreign Environment Detection (package-based) ──
# If the main app package is completely absent from the UI hierarchy,
# OR if there's a dominant foreign package and no app package, we might have lost the app.
# If our app is on screen, we trust we are in the app (even if a custom keyboard is open).
# We only trigger foreign app classification if our app is completely missing from the screen.
is_foreign = False
@@ -305,11 +349,11 @@ class SituationalAwarenessEngine:
# We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks
# for Android System UI variations across different device manufacturers.
try:
from GramAddict.core.llm_provider import query_telepathic_llm
from GramAddict.core.config import Config
screen_off = not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True)
from GramAddict.core.llm_provider import query_telepathic_llm
screen_off = not getattr(self.device.deviceV2, "info", {}).get("screenOn", True)
prompt = (
"You are a Situation Classifier for a mobile automation agent.\n"
"Analyze the given Android UI XML dump. Is this a physical DEVICE_LOCK_SCREEN, "
@@ -319,18 +363,27 @@ class SituationalAwarenessEngine:
"{\"situation\": \"OBSTACLE_LOCKED_SCREEN\" | \"OBSTACLE_SYSTEM\" | \"OBSTACLE_FOREIGN_APP\"}\n\n"
f"XML:\n{self._compress_xml(xml_dump)[:2500]}"
)
args = {}
try: args = Config().args
except Exception: pass
try:
args = Config().args
except Exception:
pass
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True)
res = query_telepathic_llm(
model=model,
url=url,
system_prompt="Strict JSON classifier.",
user_prompt=prompt,
use_local_edge=True,
)
import json
data = json.loads(res)
situ_str = data.get("situation", "")
if situ_str == "OBSTACLE_LOCKED_SCREEN":
logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: LOCKED_SCREEN.")
return SituationType.OBSTACLE_LOCKED_SCREEN
@@ -348,8 +401,9 @@ class SituationalAwarenessEngine:
# We explicitly query ScreenMemoryDB. If unknown, we ask the LLM.
# This replaces ALL brittle string/ID matching for modals.
from GramAddict.core.qdrant_memory import ScreenMemoryDB
screen_memory = ScreenMemoryDB()
compressed = self._compress_xml(xml_dump)
# ── Structural Fast-Check: Content-Creation Overlays ──
@@ -358,21 +412,21 @@ class SituationalAwarenessEngine:
# and frequently fool the LLM into thinking they are "normal" browsing.
# Detecting them structurally is O(1) and requires ZERO LLM calls.
creation_flow_markers = (
'quick_capture', # Camera / story capture overlay
'gallery_cancel_button', # Story gallery "Back to Home" button
'creation_flow', # Post creation wizard
'reel_camera', # Reel recording interface
"quick_capture", # Camera / story capture overlay
"gallery_cancel_button", # Story gallery "Back to Home" button
"creation_flow", # Post creation wizard
"reel_camera", # Reel recording interface
)
# Guard: Check against compressed string to ensure these markers ONLY appear
# as resource IDs (e.g. "id=quick_capture_...") and not as plain text in
# Guard: Check against compressed string to ensure these markers ONLY appear
# as resource IDs (e.g. "id=quick_capture_...") and not as plain text in
# user comments/bios (which would look like "text='... creation_flow ...'")
if any(re.search(rf'id=[^\s|]*{marker}', compressed, re.IGNORECASE) for marker in creation_flow_markers):
if any(re.search(rf"id=[^\s|]*{marker}", compressed, re.IGNORECASE) for marker in creation_flow_markers):
logger.info("🧠 [SAE Perceive] Content-creation overlay detected structurally → OBSTACLE_MODAL")
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
return SituationType.OBSTACLE_MODAL
cached_type = screen_memory.get_screen_type(compressed)
if cached_type:
if cached_type == "OBSTACLE_MODAL":
return SituationType.OBSTACLE_MODAL
@@ -381,9 +435,9 @@ class SituationalAwarenessEngine:
# If not cached, query LLM for autonomous structural classification
try:
from GramAddict.core.llm_provider import query_telepathic_llm
from GramAddict.core.config import Config
from GramAddict.core.llm_provider import query_telepathic_llm
prompt = (
"You are a Situation Classifier for a mobile automation agent.\n"
"Analyze the given Android UI XML dump. Is there a blocking MODAL, DIALOG, or POPUP "
@@ -394,21 +448,26 @@ class SituationalAwarenessEngine:
"or ANY content-creation flow (reel recording, post editor, live mode) is an OBSTACLE_MODAL — "
"it blocks normal navigation and must be dismissed.\n"
"Respond ONLY with a valid JSON object strictly matching this schema: "
"{\"situation\": \"OBSTACLE_MODAL\" | \"NORMAL\"}\n\n"
'{"situation": "OBSTACLE_MODAL" | "NORMAL"}\n\n'
f"XML:\n{compressed[:2500]}"
)
args = {}
try: args = Config().args
except Exception: pass
try:
args = Config().args
except Exception:
pass
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True)
res = query_telepathic_llm(
model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True
)
import json
data = json.loads(res)
situ_str = data.get("situation", "NORMAL")
if situ_str == "OBSTACLE_MODAL":
logger.info("🧠 [Smart Perceive] Screen classified as: OBSTACLE_MODAL.")
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
@@ -422,19 +481,28 @@ class SituationalAwarenessEngine:
return SituationType.NORMAL
def unlearn_current_state(self, xml_dump: str):
"""Purges the current screen's signature from Qdrant to self-heal from hallucinations."""
compressed = self._compress_xml(xml_dump)
from GramAddict.core.qdrant_memory import ScreenMemoryDB
screen_memory = ScreenMemoryDB()
screen_memory.purge_screen(compressed)
logger.info("🗑️ [Smart Perceive] Purged cached screen signature to force autonomous re-evaluation.")
# ──────────────────────────────────────────────
# 2. PLAN: AI-driven escape strategy
# ──────────────────────────────────────────────
def _plan_escape_via_llm(self, xml_dump: str, compressed: str, situation_type: SituationType, failed_actions: set = None) -> Optional[EscapeAction]:
def _plan_escape_via_llm(
self, xml_dump: str, compressed: str, situation_type: SituationType, failed_actions: set = None
) -> Optional[EscapeAction]:
"""
LLM-powered escape planning for situations where structural scan fails.
Called ONLY when recall AND structural planning both miss.
"""
from GramAddict.core.llm_provider import query_llm
from GramAddict.core.config import Config
from GramAddict.core.llm_provider import query_llm
try:
args = Config().args
@@ -455,29 +523,35 @@ class SituationalAwarenessEngine:
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
"- If nothing else works, suggest 'app_start' to force-reopen Instagram\n"
"- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n"
"- Return ONLY valid JSON: {\"action\": \"click\"|\"back\"|\"app_start\"|\"unlock\"|\"kill_foreign_apps\"|\"false_positive\", \"x\": N, \"y\": N, \"reason\": \"...\"}"
'- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}'
)
user_prompt = (
f"Situation type: {situation_type.value}\n\n"
f"Screen content:\n{compressed}\n\n"
)
user_prompt = f"Situation type: {situation_type.value}\n\n" f"Screen content:\n{compressed}\n\n"
if failed_actions:
user_prompt += f"Failed actions this session (DO NOT REPEAT): {list(failed_actions)}\n\n"
user_prompt += "What action should I take to clear this obstacle and return to Instagram? Return JSON only."
try:
resp = query_llm(url=url, model=model, prompt=user_prompt, system=system_prompt,
format_json=True, timeout=30, max_tokens=300, temperature=0.0)
resp = query_llm(
url=url,
model=model,
prompt=user_prompt,
system=system_prompt,
format_json=True,
timeout=30,
max_tokens=300,
temperature=0.0,
)
if resp and "response" in resp:
import json
data = json.loads(resp["response"])
return EscapeAction(
action_type=data.get("action", "back"),
x=int(data.get("x", 0)),
y=int(data.get("y", 0)),
reason=data.get("reason", "LLM-planned escape")
reason=data.get("reason", "LLM-planned escape"),
)
except Exception as e:
logger.warning(f"🧠 [SAE] LLM escape planning failed: {e}")
@@ -504,24 +578,24 @@ class SituationalAwarenessEngine:
logger.info(f"🔓 [SAE Act] Unlocking device: {action.reason}")
self.device.unlock()
random_sleep(1.0, 2.0)
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
app_id = getattr(self.device, "app_id", "com.instagram.android")
self.device.app_start(app_id, use_monkey=True)
random_sleep(1.5, 2.5)
elif action.action_type == "app_start":
logger.info(f"🚀 [SAE Act] Force-starting app: {action.reason}")
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
app_id = getattr(self.device, "app_id", "com.instagram.android")
self.device.app_start(app_id, use_monkey=True)
random_sleep(2.0, 3.5)
elif action.action_type == "kill_foreign_apps":
logger.info(f"🔪 [SAE Act] Killing foreign apps: {action.reason}")
# The reason string will contain the package name or 'all'
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
app_id = getattr(self.device, "app_id", "com.instagram.android")
try:
# We can dump current package again, or just get it from device
current_pkg = self.device.deviceV2.app_current().get("package")
if current_pkg and current_pkg != app_id and current_pkg not in ('com.android.systemui', 'android'):
if current_pkg and current_pkg != app_id and current_pkg not in ("com.android.systemui", "android"):
logger.info(f"🔪 Stopping {current_pkg}")
self.device.app_stop(current_pkg)
random_sleep(1.0, 2.0)
@@ -535,7 +609,7 @@ class SituationalAwarenessEngine:
logger.info(f"🏠 [SAE Act] HOME → App Start: {action.reason}")
self.device.press("home")
random_sleep(0.5, 1.0)
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
app_id = getattr(self.device, "app_id", "com.instagram.android")
self.device.app_start(app_id, use_monkey=True)
random_sleep(2.0, 3.5)
@@ -550,9 +624,10 @@ class SituationalAwarenessEngine:
Returns True if an obstacle was successfully cleared, False if already clear or failed.
"""
from GramAddict.core.exceptions import ActionBlockedError
failed_this_session = set()
cleared_something = False
last_situation = None
situation_attempts = 0
@@ -562,9 +637,9 @@ class SituationalAwarenessEngine:
xml_dump = initial_xml
else:
xml_dump = self.device.dump_hierarchy()
situation = self.perceive(xml_dump)
if last_situation != situation:
situation_attempts = 0
last_situation = situation
@@ -582,13 +657,11 @@ class SituationalAwarenessEngine:
logger.error("🚫 [SAE CRITICAL] Instagram Action Block detected! Halting to protect account.")
raise ActionBlockedError("Instagram action block detected by SAE.")
logger.warning(
f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})"
)
logger.warning(f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})")
# ── COMPRESS for memory lookup ──
compressed = self._compress_xml(xml_dump)
# ── RECALL from memory ──
recalled = self.episodes.recall(compressed)
if recalled:
@@ -597,7 +670,7 @@ class SituationalAwarenessEngine:
action = EscapeAction.from_dict(recalled)
logger.info(f"🧠 [SAE] Using recalled strategy: {action.reason}")
else:
logger.info(f"🧠 [SAE] Recalled strategy already failed this session. Using LLM planning.")
logger.info("🧠 [SAE] Recalled strategy already failed this session. Using LLM planning.")
recalled = None
if not recalled:
@@ -605,14 +678,23 @@ class SituationalAwarenessEngine:
logger.info("🧠 [SAE] Autonomous Blank Start: Escalating to LLM-assisted escape planning...")
action = self._plan_escape_via_llm(xml_dump, compressed, situation, failed_this_session)
elif situation_attempts == 3:
action = EscapeAction("app_start", reason=f"Escalation level 4: force app restart after {situation_attempts} failed attempts on this situation")
action = EscapeAction(
"app_start",
reason=f"Escalation level 4: force app restart after {situation_attempts} failed attempts on this situation",
)
else:
action = EscapeAction("home_then_app", reason=f"Nuclear escalation: HOME + app_start after {situation_attempts} failed attempts on this situation")
action = EscapeAction(
"home_then_app",
reason=f"Nuclear escalation: HOME + app_start after {situation_attempts} failed attempts on this situation",
)
# ── EXECUTE ──
if action.action_type == "false_positive":
logger.warning(f"🧠 [SAE Unlearn] LLM identified false positive obstacle. Overwriting Qdrant memory to NORMAL.")
logger.warning(
"🧠 [SAE Unlearn] LLM identified false positive obstacle. Overwriting Qdrant memory to NORMAL."
)
from GramAddict.core.qdrant_memory import ScreenMemoryDB
ScreenMemoryDB().store_screen(compressed, "NORMAL")
self._consecutive_failures = 0
return True
@@ -623,8 +705,8 @@ class SituationalAwarenessEngine:
# ── VERIFY ──
post_xml = self.device.dump_hierarchy()
post_situation = self.perceive(post_xml)
reached_normal = (post_situation == SituationType.NORMAL)
situation_changed = (post_situation != situation)
reached_normal = post_situation == SituationType.NORMAL
situation_changed = post_situation != situation
if reached_normal:
# ── LEARN FULL SUCCESS ──
@@ -635,7 +717,9 @@ class SituationalAwarenessEngine:
elif situation_changed:
# ── LEARN PARTIAL SUCCESS ──
self.episodes.learn(compressed, action, True)
logger.info(f"🔄 [SAE] Situation changed from {situation.value} to {post_situation.value}. Continuing recovery...")
logger.info(
f"🔄 [SAE] Situation changed from {situation.value} to {post_situation.value}. Continuing recovery..."
)
# We do not increment consecutive_failures or situation_attempts because we made progress
# The next loop iteration will clear failed_this_session since last_situation != situation
else:

2307
GramAddict/core/telepathic_engine.py
View File

File diff suppressed because it is too large Load Diff

1
pyproject.toml
View File

@@ -52,6 +52,7 @@ markers = [
"live: tests requiring a live ADB device",
"chaos: chaos engineering / corruption tests",
"property: hypothesis property-based tests",
"live_llm: tests requiring a live local LLM via Ollama",
]
[tool.coverage.run]

20
test_config.yml
View File

@@ -7,7 +7,7 @@
identity:
# Unter welchem Account operiert der Bot?
username: "marisaundmarc"
# Wer ist der Bot? (Wichtig für die KI-Kommentare und Profil-Analyse)
persona: "Travel blogger, landscape photographer, and outdoors enthusiast"
vibe: "friendly, authentic, helpful, and appreciative of good art"
@@ -19,14 +19,14 @@ mission:
# - stealth_lurker: Liest viel, interagiert aber nur bei extrem hoher Relevanz
# - passive_learning: "Dry-Run" Modus. Bot navigiert und lernt, führt aber NIE Aktionen aus.
strategy: "aggressive_growth"
# Wie kritisch ist der Bot bei fremden Posts? (Hoch = nur Meisterwerke, Niedrig = fast alles)
selectivity_threshold: "high"
selectivity_threshold: "high"
# Wen sucht der Bot? (Alias für target-audience)
target_audience: "travel, landscape, nature, mountain photography, wanderlust"
# persona_interests: "travel, landscape, nature" # Alternative zu target_audience
# Was hasst der Bot absolut? (Sofortiger Skip)
blacklist_topics: "onlyfans, nsfw, sale, discount, promo, 18+, giveaway, crypto"
@@ -46,17 +46,17 @@ interactions:
comment_percentage: 40 # Moderater Wert, da Kommentare "dry" sind
follow_percentage: 100 # IMMER folgen, wenn das Profil als relevant bewertet wurde
stories_percentage: 100 # IMMER Stories schauen, um menschlich zu wirken
# Detail-Limits pro Profil/Post
likes_count: "2-3" # 2-3 schnelle Likes auf dem Profil hinterlassen (sehr starkes Signal)
stories_count: "1-2" # 1-2 Stories anschauen (sehr menschliches Verhalten)
# Comment Dry Run: Wenn true, überlegt sich die AI geniale Kommentare, postet sie aber nicht in echt.
dry_run_comments: true
# Wahrscheinlichkeit (in Prozent), fremde Profile VOR dem Kommentieren tiefgründig zu analysieren
profile_learning_percentage: 100 # IMMER Profile analysieren -> Trigger für den Follow/Like Flow
# Wahrscheinlichkeit (in Prozent), das Bild visuell zu analysieren (Screenshot -> LLM), bevor interagiert wird
visual_vibe_check_percentage: 100
@@ -76,7 +76,7 @@ limits:
daily_budget_hours: 2.5
# working_hours: "09:00-21:00" # In welchem Fenster der Bot laufen darf
# time_delta_session: "60-120" # Minuten Pause zwischen Sessions
# Absolute Sicherheitsnetze pro Tag/Lauf
max_comments_per_day: 40
# total_likes_limit: 300
@@ -97,7 +97,7 @@ limits:
ignore_close_friends: true # Ignoriere alles (Posts/Stories) von "Enge Freunde"
# ── Infrastructure & System (Nur für Entwickler) ──
device: 192.168.1.206:34201
device: 192.168.1.206:42171
app-id: com.instagram.android
debug: true

89
tests/anomalies/test_hardware_anomalies.py
View File

@@ -1,9 +1,10 @@
import pytest
import os
import time
from unittest.mock import MagicMock, patch
from GramAddict.core.bot_flow import _wait_for_post_loaded, _run_zero_latency_feed_loop, FEED_MARKERS
from GramAddict.core.device_facade import DeviceFacade
import pytest
from GramAddict.core.bot_flow import FEED_MARKERS, _run_zero_latency_feed_loop, _wait_for_post_loaded
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DUMPS = {
@@ -12,14 +13,17 @@ DUMPS = {
}
FIXTURE_DIR = os.path.join(ROOT_DIR, "fixtures")
def mutate_xml_to_foreign(xml_content: str) -> str:
"""Removes meaningful text content to simulate a language failure or empty state."""
import re
# Strip text and content-desc
xml = re.sub(r'text="[^"]*"', 'text=""', xml_content)
xml = re.sub(r'content-desc="[^"]*"', 'content-desc=""', xml)
return xml
def mutate_xml_remove_feed_markers(xml_content: str) -> str:
"""Removes all feed markers to simulate a grid view or random popup."""
xml = xml_content
@@ -27,21 +31,26 @@ def mutate_xml_remove_feed_markers(xml_content: str) -> str:
xml = xml.replace(marker, "some_random_id")
return xml
class ConfigMock:
def __init__(self):
self.args = MagicMock()
self.args.interact_percentage = 0
self.args.comment_percentage = 0
@pytest.fixture
def test_dumps():
dumps = {}
with open(DUMPS["organic"], "r") as f:
dumps["post"] = f.read()
# Fake explore grid that lacks ALL feed markers
dumps["grid"] = '<?xml version="1.0"?><hierarchy><node resource-id="com.instagram.android:id/explore_grid_container" /></hierarchy>'
dumps["grid"] = (
'<?xml version="1.0"?><hierarchy><node resource-id="com.instagram.android:id/explore_grid_container" /></hierarchy>'
)
return dumps
def test_slow_loading_post_recovery(test_dumps):
"""
Test that _wait_for_post_loaded correctly handles a delay where the
@@ -50,33 +59,35 @@ def test_slow_loading_post_recovery(test_dumps):
device = MagicMock()
# Simulate: Grid -> Grid -> Error -> Post
device.dump_hierarchy.side_effect = [
test_dumps["grid"],
test_dumps["grid"],
test_dumps["grid"],
Exception("uiautomator2 temp failure"),
test_dumps["post"]
test_dumps["post"],
]
# We patch sleep to make the test super fast
with patch('GramAddict.core.bot_flow.sleep', return_value=None):
with patch("GramAddict.core.bot_flow.sleep", return_value=None):
start = time.time()
success = _wait_for_post_loaded(device, timeout=5)
# Should return true when it hits the 4th element
assert success is True
assert device.dump_hierarchy.call_count == 4
def test_wait_timeout_aborts_gracefully(test_dumps):
"""Test what happens if the network is so slow it times out entirely."""
device = MagicMock()
# Always return grid
device.dump_hierarchy.return_value = test_dumps["grid"]
# Patch time.time to simulate 6 seconds passing immediately
# We add sequence padding because python's logger internally uses time.time()
with patch('GramAddict.core.bot_flow.time.time', side_effect=[0, 1, 6, 6, 6, 6, 6, 6, 6, 6]):
with patch('GramAddict.core.bot_flow.sleep', return_value=None):
with patch("time.time", side_effect=[0, 1, 6, 6, 6, 6, 6, 6, 6, 6]):
with patch("GramAddict.core.bot_flow.sleep", return_value=None):
success = _wait_for_post_loaded(device, timeout=5)
assert success is False
def test_empty_content_extraction_guard(test_dumps):
"""
Test that if a post is loaded, but it has strange empty text (foreign language or bug),
@@ -85,38 +96,47 @@ def test_empty_content_extraction_guard(test_dumps):
device = MagicMock()
nav_graph = MagicMock()
configs = ConfigMock()
# We create a fake active inference engine to just break the loop after 1 iteration
ai = MagicMock()
# Dopamine engine controls loop exit
dopamine = MagicMock()
dopamine.is_app_session_over.side_effect = [False, True] # Run once, then exit
dopamine.is_app_session_over.side_effect = [False, True] # Run once, then exit
dopamine.wants_to_change_feed.return_value = False
dopamine.wants_to_doomscroll.return_value = False
cognitive_stack = {
"dopamine": dopamine,
"active_inference": ai,
"resonance": None, "growth_brain": None, "swarm": None, "darwin": None
"resonance": None,
"growth_brain": None,
"swarm": None,
"darwin": None,
}
# Mutate the post so it has NO text or description
broken_xml = mutate_xml_to_foreign(test_dumps["post"])
device.dump_hierarchy.return_value = broken_xml
from GramAddict.core.situational_awareness import SituationType
with patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \
patch('GramAddict.core.bot_flow.sleep'), \
patch('GramAddict.core.situational_awareness.SituationalAwarenessEngine.perceive', return_value=SituationType.NORMAL):
with (
patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll,
patch("GramAddict.core.bot_flow.sleep"),
patch(
"GramAddict.core.situational_awareness.SituationalAwarenessEngine.perceive",
return_value=SituationType.NORMAL,
),
):
result = _run_zero_latency_feed_loop(device, None, nav_graph, configs, MagicMock(), "HomeFeed", cognitive_stack)
# Ensure scroll was called (the recovery mechanism)
assert mock_scroll.called
# Check that we never called resonance evaluation because we broke early
assert not ai.predict_state.called
assert result == "FEED_EXHAUSTED"
def test_missing_feed_markers_guard(test_dumps):
"""
Test that if the UI is completely foreign (e.g., a system popup),
@@ -124,23 +144,23 @@ def test_missing_feed_markers_guard(test_dumps):
"""
device = MagicMock()
configs = ConfigMock()
dopamine = MagicMock()
dopamine.is_app_session_over.side_effect = [False, True]
dopamine.wants_to_change_feed.return_value = False
dopamine.wants_to_doomscroll.return_value = False
cognitive_stack = {"dopamine": dopamine, "growth_brain": None, "active_inference": None}
# Mutate XML to remove all FEED MARKERS
alien_xml = mutate_xml_remove_feed_markers(test_dumps["post"])
device.dump_hierarchy.return_value = alien_xml
with patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \
patch('GramAddict.core.bot_flow.sleep'):
with patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll, patch("GramAddict.core.bot_flow.sleep"):
_run_zero_latency_feed_loop(device, None, MagicMock(), configs, MagicMock(), "HomeFeed", cognitive_stack)
@patch('GramAddict.core.device_facade.u2')
@patch("GramAddict.core.device_facade.u2")
def test_xpath_watcher_initialization(mock_u2):
"""
Test fixing the critical watcher API bug.
@@ -148,21 +168,22 @@ def test_xpath_watcher_initialization(mock_u2):
"""
mock_d = MagicMock()
mock_u2.connect.return_value = mock_d
# Setup mock chain: deviceV2.watcher("crash_dialog").when(...)
mock_watcher = MagicMock()
mock_d.watcher.return_value = mock_watcher
mock_when = MagicMock()
mock_watcher.when.return_value = mock_when
# Just init the facade
from GramAddict.core.device_facade import create_device
device = create_device("fake_serial", "com.fake.app", MagicMock())
# Verify exact API call structure for XPath
mock_d.watcher.assert_any_call("crash_dialog")
mock_d.watcher.assert_any_call("system_dialog")
# We can't perfectly assert the chained arguments natively without a bit of inspection,
# but we can verify it didn't crash and called start
assert mock_d.watcher.start.called

69
tests/anomalies/test_telepathic_guards.py Normal file
View File

@@ -0,0 +1,69 @@
from GramAddict.core.telepathic_engine import TelepathicEngine
class TestTelepathicGuards:
def setup_method(self):
self.engine = TelepathicEngine()
def test_strict_story_ring_guard(self):
"""
TDD: Story rings MUST be physically near the top of the screen (y < 30%).
Post profile headers that appear further down must be aggressively blocked
when the intent is 'tap story ring avatar'.
"""
intent = "tap story ring avatar"
screen_height = 2400
# Valid Story Ring (Top of screen, but below status bar)
valid_story = {"resource_id": "reel_ring", "y": 300, "area": 100}
assert self.engine._structural_sanity_check(valid_story, intent, screen_height) is True
# Invalid Story Ring (Hallucination: Post profile header in the feed)
invalid_story = {"resource_id": "row_feed_profile_header", "y": 800, "area": 100}
assert self.engine._structural_sanity_check(invalid_story, intent, screen_height) is False
def test_strict_button_guard(self):
"""
TDD: When explicitly looking for a 'button', nodes that declare themselves
as profiles (e.g. 'go to profile') must be blocked, to prevent accidental
profile visits when clicking 'like'.
"""
intent = "Heart like button for comment"
screen_height = 2400
# Valid Like Button
valid_btn = {"resource_id": "like_button", "semantic_string": "Like", "y": 1000, "area": 100}
assert self.engine._structural_sanity_check(valid_btn, intent, screen_height) is True
# Invalid Profile Link masquerading as a match due to string proximity
invalid_prof = {
"resource_id": "username",
"semantic_string": "Go to cayleighanddavid's profile",
"y": 1000,
"area": 100,
}
assert self.engine._structural_sanity_check(invalid_prof, intent, screen_height) is False
# However, if the intent *is* profile, it should pass
intent_prof = "go to profile"
assert self.engine._structural_sanity_check(invalid_prof, intent_prof, screen_height) is True
def test_like_semantic_verification(self):
"""
TDD: Verify that 'unlike' is treated as a successful 'Like' action,
because tapping 'Like' changes the state to 'Unlike' in English Instagram.
"""
# Testing the specific regex logic inside verify_success
import re
xml_dump_success = '<node class="android.widget.ImageView" content-desc="Unlike" />'
intent = "tap like button"
marker_found = re.search(r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_success.lower())
assert marker_found is not None
xml_dump_fail = '<node class="android.widget.ImageView" content-desc="Like" />'
marker_found_fail = re.search(
r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_fail.lower()
)
assert marker_found_fail is None

95
tests/conftest.py
View File

@@ -1,130 +1,169 @@
import pytest
import logging
import os
from unittest.mock import MagicMock
import pytest
def pytest_addoption(parser):
parser.addoption(
"--live", action="store_true", default=False, help="run tests against a live ADB device (disable DeviceFacade mocks)"
"--live",
action="store_true",
default=False,
help="run tests against a live ADB device (disable DeviceFacade mocks)",
)
MagicMock.app_id = "com.instagram.android"
MagicMock._get_current_app = MagicMock(return_value="com.instagram.android")
class MockArgs:
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
class MockConfigs:
def __init__(self, args):
self.args = args
from unittest.mock import create_autospec, MagicMock
from unittest.mock import MagicMock, create_autospec
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.telepathic_engine import TelepathicEngine
def create_mock_device():
mock = create_autospec(DeviceFacade, instance=True)
mock.app_id = "com.instagram.android"
mock.device_id = "test_device"
mock.info = {"displayWidth": 1080, "displayHeight": 2400}
mock.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400}
mock.cm_to_pixels.side_effect = lambda cm: int(cm * 10)
mock.shell.return_value = "" # Ensure SendEventInjector detection gets a string
import uuid
mock.dump_hierarchy.side_effect = lambda: f"<hierarchy><node resource-id=\"com.instagram.android:id/row_feed_photo_profile_name\" bounds=\"[0,200][1080,260]\" text=\"testuser\" /><node resource-id=\"com.instagram.android:id/row_comment_imageview\" bounds=\"[10,10][20,20]\" content-desc=\"Story\" text=\"following\" /><node resource-id=\"com.instagram.android:id/button_like\" bounds=\"[50,50][60,60]\" /><node resource-id=\"com.instagram.android:id/reel_viewer\" /><node sid=\"{uuid.uuid4()}\" /></hierarchy>"
mock.dump_hierarchy.side_effect = (
lambda: f'<hierarchy><node resource-id="com.instagram.android:id/row_feed_photo_profile_name" bounds="[0,200][1080,260]" text="testuser" /><node resource-id="com.instagram.android:id/row_comment_imageview" bounds="[10,10][20,20]" content-desc="Story" text="following" /><node resource-id="com.instagram.android:id/button_like" bounds="[50,50][60,60]" /><node resource-id="com.instagram.android:id/reel_viewer" /><node sid="{uuid.uuid4()}" /></hierarchy>'
)
return mock
def create_mock_telepathic_engine():
mock = create_autospec(TelepathicEngine, instance=True)
mock.find_best_node.return_value = {"x": 500, "y": 500, "confidence": 0.9}
mock.evaluate_profile_vibe.return_value = {"quality_score": 8, "matches_niche": True, "reason": "Mocked positive vibe"}
mock.evaluate_grid_visuals.return_value = {"x": 500, "y": 500, "score": 0.99, "semantic": "Mocked matching grid cell", "source": "vlm_grid"}
mock._extract_semantic_nodes.return_value = [{"x": 500, "y": 500, "semantic_string": "dummy node"}]
mock.evaluate_profile_vibe.return_value = {
"quality_score": 8,
"matches_niche": True,
"reason": "Mocked positive vibe",
}
mock.evaluate_grid_visuals.return_value = {
"x": 500,
"y": 500,
"score": 0.99,
"semantic": "Mocked matching grid cell",
"source": "vlm_grid",
}
mock.find_best_node.return_value = {"x": 500, "y": 500, "semantic_string": "dummy node"}
return mock
@pytest.fixture
def mock_logger():
return logging.getLogger("test")
@pytest.fixture
def device(request):
if request.config.getoption("--live"):
from GramAddict.core.device_facade import create_device
import yaml
import os
import yaml
from GramAddict.core.device_facade import create_device
device_id = "emulator-5554"
app_id = "com.instagram.android"
config_path = "test_config.yml"
if os.path.exists(config_path):
try:
with open(config_path, 'r', encoding='utf-8') as f:
with open(config_path, "r", encoding="utf-8") as f:
config = yaml.safe_load(f)
if config:
device_id = config.get("device", device_id)
app_id = config.get("app-id", app_id)
except Exception as e:
print(f"⚠️ Warning: Could not load {config_path}: {e}")
print(f"🚀 Connecting to live device: {device_id} (App: {app_id})")
return create_device(device_id, app_id)
return create_mock_device()
@pytest.fixture(autouse=True)
def reset_singletons():
"""Ensure all core engine singletons are fresh for each test."""
from GramAddict.core.telepathic_engine import TelepathicEngine
from GramAddict.core.goap import GoalExecutor
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
from GramAddict.core.qdrant_memory import QdrantBase
from GramAddict.core.behaviors import PluginRegistry
from GramAddict.core.goap import GoalExecutor
from GramAddict.core.physics.biomechanics import PhysicsBody
from GramAddict.core.physics.sendevent_injector import SendEventInjector
from GramAddict.core.qdrant_memory import QdrantBase
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
from GramAddict.core.telepathic_engine import TelepathicEngine
TelepathicEngine.reset()
GoalExecutor.reset()
SituationalAwarenessEngine.reset()
PluginRegistry.reset()
PhysicsBody.reset()
SendEventInjector.reset()
QdrantBase._connection_failed_logged = False
from GramAddict.core.dojo_engine import DojoEngine
if hasattr(DojoEngine, "reset"):
DojoEngine.reset()
else:
DojoEngine._instance = None
# Aggressively wipe on-disk session files to prevent state leakage in tests
for f in ["telepathic_memory.json", "telepathic_blacklist.json", "growth_brain_memory.json", "gramaddict_nav_map.json", "l2_channels_cache.json"]:
for f in [
"telepathic_memory.json",
"telepathic_blacklist.json",
"growth_brain_memory.json",
"gramaddict_nav_map.json",
"l2_channels_cache.json",
]:
if os.path.exists(f):
try:
os.remove(f)
except Exception:
pass
yield
# Post-test cleanup
PhysicsBody.reset()
SendEventInjector.reset()
@pytest.fixture(autouse=True)
def telepathic_mock(monkeypatch, request):
if request.config.getoption("--live"):
# TelepathicEngine is a singleton, allow it to run natively
return None
import GramAddict.core.telepathic_engine
engine = create_mock_telepathic_engine()
monkeypatch.setattr(GramAddict.core.telepathic_engine.TelepathicEngine, "get_instance", lambda: engine)
return engine
@pytest.fixture
def mock_cognitive_stack():
stack = {
@@ -138,7 +177,7 @@ def mock_cognitive_stack():
"nav_graph": MagicMock(),
"zero_engine": MagicMock(),
"crm": MagicMock(),
"telepathic": create_mock_telepathic_engine()
"telepathic": create_mock_telepathic_engine(),
}
stack["radome"].sanitize_xml.side_effect = lambda x: x
return stack

151
tests/e2e/conftest.py
View File

@@ -1,11 +1,13 @@
import sys
import os
import pytest
import sys
import time
from unittest.mock import MagicMock
import pytest
from GramAddict.core import utils
# Force Qdrant mocking globally across ALL E2E tests so we never
# Force Qdrant mocking globally across ALL E2E tests so we never
# block on connection refused trying to hit localhost:6344
mock_qdrant = MagicMock()
@@ -16,11 +18,12 @@ mock_qdrant.get_collection.return_value = mock_collection
sys.modules["qdrant_client"].QdrantClient = MagicMock(return_value=mock_qdrant)
@pytest.fixture
def e2e_device_dump_injector(request):
"""
Provides a factory to mock device.dump_hierarchy using real XML files.
Will gracefully fail with a comprehensive assertion if the file is missing
Will gracefully fail with a comprehensive assertion if the file is missing
(per 'ECHTE DUMPS fehlen' reporting requirement).
"""
if request.config.getoption("--live"):
@@ -29,30 +32,36 @@ def e2e_device_dump_injector(request):
def _inject_dump(device_mock, xml_filename):
fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
xml_path = os.path.join(fix_dir, xml_filename)
if not os.path.exists(xml_path):
pytest.fail(f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.", pytrace=False)
pytest.fail(
f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.",
pytrace=False,
)
with open(xml_path, "r") as f:
real_xml = f.read()
device_mock.dump_hierarchy.return_value = real_xml
return real_xml
return _inject_dump
class VirtualClock:
def __init__(self):
self.time = 0.0
self.animation_target_time = 0.0
def sleep(self, seconds):
if hasattr(seconds, '__iter__'):
return # For edge case where something weird is passed
if hasattr(seconds, "__iter__"):
return # For edge case where something weird is passed
self.time += float(seconds)
clock = VirtualClock()
@pytest.fixture
def dynamic_e2e_dump_injector(monkeypatch, request):
"""
@@ -66,9 +75,9 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
def _inject(device_mock, state_map, initial_xml):
from GramAddict.core.q_nav_graph import QNavGraph
fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
def load_xml(filename):
path = os.path.join(fix_dir, filename)
if not os.path.exists(path):
@@ -79,47 +88,56 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
# History stack to allow "back" navigation
device_mock._xml_history = [load_xml(initial_xml)]
device_mock._current_active_xml = device_mock._xml_history[-1]
import uuid
def _dump_hierarchy_hook():
if clock.time < clock.animation_target_time:
pytest.fail(f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! "
f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. "
f"Add a time.sleep() guard before interacting with the UI after a click.", pytrace=False)
pytest.fail(
f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! "
f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. "
f"Add a time.sleep() guard before interacting with the UI after a click.",
pytrace=False,
)
xml = device_mock._current_active_xml
if xml and "</hierarchy>" in xml:
xml = xml.replace("</hierarchy>", f"<node sid=\"{uuid.uuid4()}\" /></hierarchy>")
xml = xml.replace("</hierarchy>", f'<node sid="{uuid.uuid4()}" /></hierarchy>')
return xml
device_mock.dump_hierarchy.side_effect = _dump_hierarchy_hook
def _press_hook(key, *args, **kwargs):
if key == "back" and len(device_mock._xml_history) > 1:
device_mock._xml_history.pop()
device_mock._current_active_xml = device_mock._xml_history[-1]
clock.animation_target_time = clock.time + 1.5
device_mock.press.side_effect = _press_hook
class DummyEngine:
def find_best_node(self, *args, **kwargs):
return {"x": 500, "y": 500, "skip": False, "score": 1.0, "source": "e2e_mock"}
def verify_success(self, *args, **kwargs):
return True
def confirm_click(self, *args, **kwargs):
pass
def reject_click(self, *args, **kwargs):
pass
original_execute = QNavGraph._execute_transition
from GramAddict.core.goap import GoalExecutor
original_goap_execute = GoalExecutor._execute_action
def _mock_execute_transition(nav_self, action, zero_engine=None, max_retries=2):
if action == 'tap_post_username':
if action == "tap_post_username":
return True
original_click = nav_self.device.click
def _click_hook(obj=None, *args, **kwargs):
original_click(obj, *args, **kwargs)
if action in state_map:
@@ -127,22 +145,24 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
device_mock._xml_history.append(new_xml)
device_mock._current_active_xml = new_xml
clock.animation_target_time = clock.time + 1.5
nav_self.device.click = _click_hook
try:
success = original_execute(nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries)
success = original_execute(
nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries
)
return success
finally:
nav_self.device.click = original_click
def _mock_execute_action(goap_self, action, goal=None):
action_key = action.replace(" ", "_")
if action_key == 'tap_post_username':
if action_key == "tap_post_username":
return True
original_click = goap_self.device.click
def _click_hook(obj=None, *args, **kwargs):
original_click(obj, *args, **kwargs)
if action_key in state_map:
@@ -155,20 +175,21 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
device_mock._xml_history.append(new_xml)
device_mock._current_active_xml = new_xml
clock.animation_target_time = clock.time + 1.5
goap_self.device.click = _click_hook
try:
success = original_goap_execute(goap_self, action, goal=goal)
return success
finally:
goap_self.device.click = original_click
monkeypatch.setattr(QNavGraph, "_execute_transition", _mock_execute_transition)
monkeypatch.setattr(GoalExecutor, "_execute_action", _mock_execute_action)
return _inject
@pytest.fixture(autouse=True)
def mock_all_delays(monkeypatch, request):
"""
@@ -180,49 +201,72 @@ def mock_all_delays(monkeypatch, request):
return
global clock
clock.time = 0.0 # reset for test
clock.time = 0.0 # reset for test
clock.animation_target_time = 0.0
def simulate_sleep(seconds):
clock.sleep(seconds)
money_sleep = lambda x: simulate_sleep(x)
random_sleep = lambda *args, **kwargs: simulate_sleep(1.0) # Assume 1.0 minimum for randoms
random_sleep = lambda a=1.0, b=2.0, *args, **kwargs: simulate_sleep(max(1.5, float(a)))
monkeypatch.setattr(time, "sleep", money_sleep)
monkeypatch.setattr(utils, "random_sleep", random_sleep)
monkeypatch.setattr(utils, "sleep", money_sleep)
# Needs to capture specific module sleeps depending on how they imported it
try:
from GramAddict.core import bot_flow
monkeypatch.setattr(bot_flow, "sleep", money_sleep)
monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound
monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound
if hasattr(bot_flow, "random_sleep"):
monkeypatch.setattr(bot_flow, "random_sleep", random_sleep)
from GramAddict.core import q_nav_graph
monkeypatch.setattr(q_nav_graph.random, "uniform", lambda a, b: float(a))
if hasattr(q_nav_graph, "random_sleep"):
monkeypatch.setattr(q_nav_graph, "random_sleep", random_sleep)
from GramAddict.core import goap
if hasattr(goap, "random"):
monkeypatch.setattr(goap.random, "uniform", lambda a, b: float(a))
if hasattr(goap, "random_sleep"):
monkeypatch.setattr(goap, "random_sleep", random_sleep)
monkeypatch.setattr(utils.random, "uniform", lambda a, b: float(a))
from GramAddict.core import device_facade
monkeypatch.setattr(device_facade, "sleep", money_sleep)
monkeypatch.setattr(device_facade.random, "uniform", lambda a, b: float(a))
except Exception:
pass
if hasattr(device_facade, "random_sleep"):
monkeypatch.setattr(device_facade, "random_sleep", random_sleep)
except Exception as e:
print(f"Mocking delays exception: {e}")
# Standardize DarwinEngine across tests to prevent mockup math errors on session end
try:
from GramAddict.core.darwin_engine import DarwinEngine
monkeypatch.setattr(DarwinEngine, "evaluate_session_end", lambda *args, **kwargs: None)
except ImportError:
pass
@pytest.fixture(autouse=True)
def mock_identity_guard(monkeypatch):
import GramAddict.core.bot_flow
monkeypatch.setattr(GramAddict.core.bot_flow, "verify_and_switch_account", lambda *args, **kwargs: True)
@pytest.fixture
def e2e_configs():
import argparse
configs = MagicMock()
configs.username = "testuser"
configs.args = argparse.Namespace(
@@ -254,6 +298,7 @@ def e2e_configs():
)
return configs
@pytest.fixture(autouse=True)
def mock_sae_perceive(request, monkeypatch):
"""
@@ -263,9 +308,15 @@ def mock_sae_perceive(request, monkeypatch):
"""
if "test_e2e_sae.py" in str(request.node.fspath):
return
if "test_e2e_real_llm_learning.py" in str(request.node.fspath):
return
if request.config.getoption("--live"):
return
import GramAddict.core.situational_awareness
monkeypatch.setattr(GramAddict.core.situational_awareness.SituationalAwarenessEngine, "perceive", lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL)
import GramAddict.core.situational_awareness
monkeypatch.setattr(
GramAddict.core.situational_awareness.SituationalAwarenessEngine,
"perceive",
lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL,
)

92
tests/e2e/test_e2e_config_goal_limits.py Normal file
View File

@@ -0,0 +1,92 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
from GramAddict.core.session_state import SessionState
def test_feed_loop_respects_config_limits(device, mock_cognitive_stack):
"""
Testet, ob die Config (Ziele/Limits) beachtet wird:
Erreicht der Bot sein Ziel (z.B. total_likes_limit) und stoppt er dann?
"""
# 1. Simulate dopamine so we don't naturally exit early due to session time
mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack[
"resonance"
].calculate_resonance.return_value = 0.75 # < 0.8 to avoid rabbit hole, but high enough to engage
# 2. Setup Config mimicking test_config.yml goals
configs = MagicMock()
configs.args.total_likes_limit = 2
configs.args.end_if_likes_limit_reached = True
configs.args.interact_percentage = 100
configs.args.likes_percentage = 100
configs.args.follow_percentage = 0
configs.args.comment_percentage = 0
configs.args.visual_vibe_check_percentage = 0
configs.args.profile_learning_percentage = 0
configs.args.repost_percentage = 0
# 3. Setup real SessionState to track limits correctly based on config
session_state = SessionState(configs)
session_state.set_limits_session()
# 4. Provide a UI dump that has content so the bot interacts
device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_button_like" />
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="test_user" />
<node resource-id="com.instagram.android:id/row_feed_photo_imageview" content-desc="test image" />
</hierarchy>"""
# Prevent radome from stripping our mock structure
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
mock_cognitive_stack["nav_graph"].do.return_value = True
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract,
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll"),
patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}),
patch("GramAddict.core.bot_flow._humanized_click") as mock_click,
patch("GramAddict.core.bot_flow.sleep"),
patch("GramAddict.core.bot_flow.random.random", return_value=0.1),
): # Force pass probabilities
mock_extract.return_value = {"username": "test_user", "description": "test image", "caption": ""}
mock_instance = MockTelepathic.get_instance.return_value
# Nodes for standard flow
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}]
# When finding the like button
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_cognitive_stack["telepathic"] = mock_instance
# We'll patch `_humanized_click` to increment the like counter to simulate the interaction succeeding.
def mock_click_side_effect(*args, **kwargs):
session_state.totalLikes += 1
session_state.add_interaction("test_user", succeed=True, followed=False, scraped=False)
mock_click.side_effect = mock_click_side_effect
# Run the autonomous loop
result = _run_zero_latency_feed_loop(
device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
# 5. Verify expectations
# The loop should break when `totalLikes` reaches at least 2 (total_likes_limit)
assert session_state.totalLikes >= 2, f"Expected at least 2 likes, got {session_state.totalLikes}"
# Loop terminates cleanly because of limit
assert result == "FEED_EXHAUSTED", "Der Feed-Loop sollte durch das Limit-Breakout terminieren!"

47
tests/e2e/test_e2e_navigation_escape_dm_trap.py
View File

@@ -42,9 +42,11 @@ Expected Behaviour After Green Phase
3. ``TelepathicEngine.find_best_node()`` with a profile-grid intent returns ``None``
(or a ``{"blocked_by_dm_thread": True}`` sentinel) when the XML is a DM thread.
"""
import os
from unittest.mock import MagicMock
import pytest
from unittest.mock import MagicMock, patch, PropertyMock
# ──────────────────────────────────────────────
# Fixture Helpers
@@ -65,14 +67,12 @@ def _load_fixture(filename: str) -> str:
return f.read()
# ──────────────────────────────────────────────
# Test 3: Structural Guard — TelepathicEngine must refuse to find
# profile-intent nodes inside a DM thread
# ──────────────────────────────────────────────
class TestTelepathicEngineDmForbiddenZone:
"""
RED: When the visible XML is a DM thread and the intent is profile-related
@@ -86,25 +86,15 @@ class TestTelepathicEngineDmForbiddenZone:
"""
def _make_engine(self):
with patch("GramAddict.core.telepathic_engine.QdrantBase") as MockQdrant, \
patch("GramAddict.core.telepathic_engine.query_telepathic_llm"), \
patch("GramAddict.core.telepathic_engine.dump_ui_state"):
from GramAddict.core.telepathic_engine import TelepathicEngine
e = TelepathicEngine.__new__(TelepathicEngine)
e._embedding_cache = {}
e._intent_cache = {}
e._blacklist = {}
e._memory = {}
e._cached_username = "testuser"
e._cached_app_id = "com.instagram.android"
# Mock embedding_helper so vector stage is a no-op (returns None → falls to VLM)
mock_helper = MagicMock()
mock_helper._get_embedding.return_value = None
e.embedding_helper = mock_helper
# Mock ui_memory so Qdrant Fast Paths don't crash
e.ui_memory = MagicMock()
e.ui_memory.retrieve_memory.return_value = None
# We only need a raw TelepathicEngine instance
from GramAddict.core.telepathic_engine import TelepathicEngine
TelepathicEngine._instance = None
e = TelepathicEngine()
# Mock the internal resolver's LLM call to prevent actual OLLAMA requests during fast-paths
e._resolver.resolve = MagicMock(return_value=None)
return e
def test_profile_intent_is_blocked_when_dm_thread_is_active(self):
@@ -128,10 +118,7 @@ class TestTelepathicEngineDmForbiddenZone:
]
for intent in profile_seeking_intents:
# Patch embedding to None so vector stage is a no-op; VLM path also mocked off
with patch.object(engine, "_get_cached_embedding", return_value=None), \
patch.object(engine, "_vision_cortex_fallback", return_value=None):
result = engine.find_best_node(dm_xml, intent, device=device)
result = engine.find_best_node(dm_xml, intent, device=device)
# The keyword fast-path WILL find nodes in the DM thread (e.g. the 'view_profile_button'
# has 'profile' in its resource-id, matching the intent). The guard must intercept
@@ -162,10 +149,7 @@ class TestTelepathicEngineDmForbiddenZone:
# This intent is used by dm_engine.py to find the message composer
dm_intent = "find the message input text field"
# Mock the embedding calls so we don't block on Qdrant during unit test
with patch.object(engine, "_get_cached_embedding", return_value=None), \
patch.object(engine, "_vision_cortex_fallback", return_value=None):
result = engine.find_best_node(dm_xml, dm_intent, device=device)
result = engine.find_best_node(dm_xml, dm_intent, device=device)
# Should NOT be blocked — DM intents are valid inside a DM thread
# (may be None if keyword/vector stage misses, but must NOT be blocked_by_dm_thread)
@@ -174,4 +158,3 @@ class TestTelepathicEngineDmForbiddenZone:
f"DM intent '{dm_intent}' was incorrectly blocked inside a DM thread. "
f"The structural guard must only block PROFILE-seeking intents."
)

163
tests/e2e/test_e2e_real_llm_learning.py Normal file
View File

@@ -0,0 +1,163 @@
"""
Real LLM + Qdrant Integration Test
Tests the extreme learning behavior of the autonomous engine by hitting
the real local Ollama instance and storing/retrieving from local Qdrant.
Requirements:
- Ollama must be running on localhost:11434
- llama3.2-vision must be available locally
- Qdrant must be running locally
"""
import time
import uuid
from unittest.mock import MagicMock, patch
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.qdrant_memory import ScreenMemoryDB
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
# ─────────────────────────────────────────────────────
# Test Setup & Isolation
# ─────────────────────────────────────────────────────
@pytest.fixture(scope="module")
def isolated_screen_memory():
"""Ensures we use a separate Qdrant collection for real LLM testing and clean it."""
# We patch __init__ so that any instantiation uses the test collection
original_init = ScreenMemoryDB.__init__
def test_init(self):
super(ScreenMemoryDB, self).__init__(collection_name="test_real_llm_screens")
ScreenMemoryDB.__init__ = test_init
db = ScreenMemoryDB()
if db.is_connected:
db.wipe_collection()
yield db
# Restore original
ScreenMemoryDB.__init__ = original_init
def make_mock_device(app_id="com.instagram.android"):
device = MagicMock(spec=DeviceFacade)
device.app_id = app_id
device.deviceV2 = MagicMock()
device.dump_hierarchy = MagicMock()
device.click = MagicMock()
device.press = MagicMock()
device.app_start = MagicMock()
device._trace_counter = 0
device._trace_dir = "/tmp/test_traces"
return device
# ─────────────────────────────────────────────────────
# Tests
# ─────────────────────────────────────────────────────
@pytest.mark.live_llm
def test_real_llm_learning_and_unlearning(isolated_screen_memory):
"""
Testet das echte Lernverhalten:
1. Pass: Unbekanntes XML -> LLM wird angefragt -> Speichert in Qdrant
2. Pass: Gleiches XML -> LLM wird NICHT angefragt -> Holt aus Qdrant
3. Pass (Unlearn): Wir löschen den State (Simulation Fehler) -> Gleiches XML -> LLM wird wieder angefragt
"""
# Check if Qdrant is connected. If not, we skip the test gracefully.
if not isolated_screen_memory.is_connected:
pytest.skip("Qdrant is not running locally. Skipping live integration test.")
# Generate completely unique XML so it's guaranteed NOT in any cache
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"REAL_LLM_TEST_{uuid.uuid4().hex[:8]}"
# A simple modal to trigger perception
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="Dismiss" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[100,2000][540,2100]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# We patch the underlying LLM call just to spy on it (wraps the original function)
from GramAddict.core.llm_provider import query_telepathic_llm
with patch("GramAddict.core.llm_provider.query_telepathic_llm", wraps=query_telepathic_llm) as spy_llm:
# ---------------------------------------------------------
# PASS 1: The Initial Encounter (Learn)
# ---------------------------------------------------------
print(f"\n--- PASS 1: Querying real LLM for '{random_text}' ---")
start_time = time.time()
# This will block and hit the real local Ollama
result_pass1 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 1 completed in {duration:.2f}s. Result: {result_pass1}")
# Assertions
assert spy_llm.call_count == 1, "LLM was not called on unknown XML!"
assert result_pass1 in [
SituationType.OBSTACLE_MODAL,
SituationType.NORMAL,
SituationType.OBSTACLE_FOREIGN_APP,
], "Invalid LLM perception result"
spy_llm.reset_mock()
# Give Qdrant a split second to index the new point
time.sleep(0.5)
# ---------------------------------------------------------
# PASS 2: The Recall (Cache Hit)
# ---------------------------------------------------------
print("\\n--- PASS 2: Recalling from Qdrant ---")
start_time = time.time()
result_pass2 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 2 completed in {duration:.2f}s. Result: {result_pass2}")
# Assertions
assert spy_llm.call_count == 0, "LLM was called again despite being in Qdrant!"
assert result_pass2 == result_pass1, "Qdrant cache returned a different result than the initial LLM call!"
assert duration < 1.0, f"Qdrant retrieval took too long ({duration:.2f}s). Should be sub-second."
# ---------------------------------------------------------
# PASS 3: The Unlearn (Mistake Recovery)
# ---------------------------------------------------------
print("\\n--- PASS 3: Unlearning and verifying re-query ---")
# We simulate that the bot decided this classification was wrong and unlearns it
sae.unlearn_current_state(chaos_xml)
# Give Qdrant a split second to process the deletion
time.sleep(0.5)
start_time = time.time()
result_pass3 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 3 completed in {duration:.2f}s. Result: {result_pass3}")
# Assertions
assert spy_llm.call_count == 1, "LLM was NOT called after unlearning! Qdrant deletion failed."
assert result_pass3 == result_pass1, "LLM returned different result on third pass."
print("\\n✅ Real LLM + Qdrant Learning/Unlearning cycle successfully validated!")

254
tests/e2e/test_e2e_sae.py
View File

@@ -4,89 +4,108 @@ Tests autonomous recovery from foreign apps, unknown modals, and learning.
Uses REAL XML dumps from production sessions.
"""
import pytest
import os
from unittest.mock import MagicMock, patch
from GramAddict.core.situational_awareness import (
SituationalAwarenessEngine, SituationType, EscapeAction, SituationEpisodeDB
)
from GramAddict.core.device_facade import DeviceFacade
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.situational_awareness import (
EscapeAction,
SituationalAwarenessEngine,
SituationType,
)
# ─────────────────────────────────────────────────────
# Test Fixtures: Real-world XML scenarios
# ─────────────────────────────────────────────────────
@pytest.fixture(autouse=True)
def mock_screen_memory():
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None), \
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"):
with (
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None),
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"),
):
yield
@pytest.fixture(autouse=True)
def mock_telepathic_classifier():
with patch("GramAddict.core.llm_provider.query_telepathic_llm") as mock_llm:
def side_effect(model, url, system_prompt, user_prompt, use_local_edge):
if "keyguard_status_view" in user_prompt or "lock_icon" in user_prompt:
return '{"situation": "OBSTACLE_LOCKED_SCREEN"}'
elif "permissioncontroller" in user_prompt:
return '{"situation": "OBSTACLE_SYSTEM"}'
# If it's a passive scaffold but no active modal markers, it's NORMAL
is_passive_only = "bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt
if "survey_overlay_container" in user_prompt or "mystery_interstitial_container" in user_prompt or ("bottom_sheet_container" in user_prompt and not is_passive_only):
is_passive_only = (
"bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt
)
if (
"survey_overlay_container" in user_prompt
or "mystery_interstitial_container" in user_prompt
or ("bottom_sheet_container" in user_prompt and not is_passive_only)
):
return '{"situation": "OBSTACLE_MODAL"}'
elif "feed_tab" in user_prompt:
return '{"situation": "NORMAL"}'
else:
return '{"situation": "OBSTACLE_FOREIGN_APP"}'
mock_llm.side_effect = side_effect
yield mock_llm
@pytest.fixture(autouse=True)
def mock_fallback_llm():
with patch("GramAddict.core.llm_provider.query_llm") as mock_llm:
def side_effect(*args, **kwargs):
prompt = kwargs.get('prompt', args[2] if len(args) > 2 else "")
prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "")
prompt_lower = prompt.lower()
if "obstacle_foreign_app" in prompt_lower:
return {"response": '{"action": "kill_foreign_apps", "x": 0, "y": 0, "reason": "Killing foreign app"}'}
elif "obstacle_locked_screen" in prompt_lower:
return {"response": '{"action": "unlock", "x": 0, "y": 0, "reason": "Unlocking device"}'}
elif "close_friends" in prompt_lower:
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Safe fallback for follow sheet"}'}
# Simulate LLM preferring BACK first for modals/dialogs
if "back:0,0" not in prompt_lower:
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Trying safe BACK first"}'}
if "not now" in prompt_lower or "später" in prompt_lower or "deny" in prompt_lower:
return {"response": '{"action": "click", "x": 320, "y": 1850, "reason": "Found dismiss button"}'}
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback to back"}'}
mock_llm.side_effect = side_effect
yield mock_llm
GOOGLE_SEARCH_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
GOOGLE_SEARCH_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.google.android.googlequicksearchbox" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.google.android.googlequicksearchbox:id/search_box" class="android.widget.EditText" package="com.google.android.googlequicksearchbox" content-desc="Search" clickable="true" bounds="[50,200][1030,300]" />
<node index="1" text="Close" resource-id="com.google.android.googlequicksearchbox:id/close_button" class="android.widget.ImageButton" package="com.google.android.googlequicksearchbox" content-desc="Close" clickable="true" bounds="[980,200][1050,280]" />
</node>
</hierarchy>'''
</hierarchy>"""
INSTAGRAM_HOME_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
INSTAGRAM_HOME_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" selected="true" bounds="[0,2235][216,2361]" />
<node index="1" text="" resource-id="com.instagram.android:id/search_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Search and Explore" clickable="true" bounds="[216,2235][432,2361]" />
<node index="2" text="" resource-id="com.instagram.android:id/profile_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Profile" clickable="true" bounds="[864,2235][1080,2361]" />
</node>
</hierarchy>'''
</hierarchy>"""
INSTAGRAM_SURVEY_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
INSTAGRAM_SURVEY_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" bounds="[0,2235][216,2361]" />
@@ -96,9 +115,9 @@ INSTAGRAM_SURVEY_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes'
<node text="Take Survey" resource-id="com.instagram.android:id/button_positive" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[540,1800][980,1900]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
UNKNOWN_MODAL_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
UNKNOWN_MODAL_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" bounds="[0,2235][216,2361]" />
@@ -108,9 +127,9 @@ UNKNOWN_MODAL_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<node text="Jetzt ansehen" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[540,2000][980,2100]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
PERMISSION_DIALOG_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
PERMISSION_DIALOG_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.permissioncontroller" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="com.android.permissioncontroller:id/grant_dialog" class="android.widget.LinearLayout" package="com.android.permissioncontroller" clickable="false" bounds="[100,800][980,1600]">
@@ -119,9 +138,9 @@ PERMISSION_DIALOG_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes'
<node text="Allow" resource-id="com.android.permissioncontroller:id/permission_allow_button" class="android.widget.Button" package="com.android.permissioncontroller" clickable="true" bounds="[550,1400][930,1500]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
LOCK_SCREEN_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
LOCK_SCREEN_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.systemui" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.android.systemui:id/keyguard_status_view" class="android.widget.FrameLayout" package="com.android.systemui" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
@@ -129,13 +148,14 @@ LOCK_SCREEN_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
</node>
<node index="1" text="" resource-id="com.android.systemui:id/lock_icon" class="android.widget.ImageView" package="com.android.systemui" content-desc="Lock icon" clickable="true" bounds="[490,2100][590,2200]" />
</node>
</hierarchy>'''
</hierarchy>"""
# ─────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────
def make_mock_device(app_id="com.instagram.android"):
device = MagicMock(spec=DeviceFacade)
device.app_id = app_id
@@ -154,6 +174,7 @@ def make_mock_device(app_id="com.instagram.android"):
# PERCEPTION TESTS
# ─────────────────────────────────────────────────────
class TestSAEPerception:
"""Tests that the SAE correctly classifies screen situations."""
@@ -171,6 +192,7 @@ class TestSAEPerception:
def test_perceive_notification_shade(self):
import os
dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml")
try:
with open(dump_path, "r") as f:
@@ -180,7 +202,7 @@ class TestSAEPerception:
result = sae.perceive(shade_xml)
assert result == SituationType.OBSTACLE_FOREIGN_APP
except FileNotFoundError:
pass # allow test format to compile if fixture accidentally not available
pass # allow test format to compile if fixture accidentally not available
def test_perceive_system_permission_dialog(self):
device = make_mock_device()
@@ -201,10 +223,52 @@ class TestSAEPerception:
result = sae.perceive(UNKNOWN_MODAL_XML)
assert result == SituationType.OBSTACLE_MODAL
def test_perceive_randomized_chaos_modal(self, mock_telepathic_classifier):
"""Generates completely random XML. Proves SAE passes dynamic state to VLM without hardcoded heuristics."""
import uuid
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"Nonsense_Text_{uuid.uuid4().hex[:8]}"
random_button_text = f"Dismiss_{uuid.uuid4().hex[:8]}"
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="{random_button_text}" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[100,2000][540,2100]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# Override the mock behavior locally for this test to return OBSTACLE_MODAL
def local_side_effect(model, url, system_prompt, user_prompt, use_local_edge):
if random_text in user_prompt:
return '{"situation": "OBSTACLE_MODAL"}'
return '{"situation": "NORMAL"}'
mock_telepathic_classifier.side_effect = local_side_effect
result = sae.perceive(chaos_xml)
assert result == SituationType.OBSTACLE_MODAL
# PROOF: The VLM was actually called, and the prompt contained our randomized strings!
mock_telepathic_classifier.assert_called_once()
_, kwargs = mock_telepathic_classifier.call_args
user_prompt = kwargs.get("user_prompt", "")
id_suffix = random_id.split("/")[-1]
assert id_suffix in user_prompt, "Bot did not pass the random ID to VLM!"
assert random_text in user_prompt, "Bot did not pass the random text to VLM!"
assert random_button_text in user_prompt, "Bot did not pass the random button text to VLM!"
def test_perceive_action_blocked(self):
blocked_xml = INSTAGRAM_HOME_XML.replace(
'text="" resource-id="com.instagram.android:id/feed_tab"',
'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"'
'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"',
)
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
@@ -227,13 +291,13 @@ class TestSAEPerception:
"""Passive scaffold containers (bottom_sheet_container_view, bottom_sheet_camera_container) must NOT be OBSTACLE_MODAL."""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# XML containing navigation tabs + the passive scaffold container
passive_xml = INSTAGRAM_HOME_XML.replace(
'<node index="1" text="" resource-id="com.instagram.android:id/main_feed_container"',
'<node index="1" text="" resource-id="com.instagram.android:id/bottom_sheet_container_view" />\n'
'<node index="2" text="" resource-id="com.instagram.android:id/bottom_sheet_camera_container" />\n'
'<node index="3" text="" resource-id="com.instagram.android:id/main_feed_container"'
'<node index="3" text="" resource-id="com.instagram.android:id/main_feed_container"',
)
result = sae.perceive(passive_xml)
assert result == SituationType.NORMAL, f"Passive scaffold misclassified as {result}"
@@ -312,11 +376,11 @@ class TestSAERealFixturePerception:
assert result == SituationType.OBSTACLE_MODAL, f"Mystery interstitial misclassified as {result}"
# ─────────────────────────────────────────────────────
# FULL AUTONOMOUS RECOVERY TESTS
# ─────────────────────────────────────────────────────
class TestSAEAutonomousRecovery:
"""Tests the full perceive→plan→act→verify→learn loop."""
@@ -329,8 +393,7 @@ class TestSAEAutonomousRecovery:
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.app_start.assert_called_with("com.instagram.android", use_monkey=True)
@@ -339,13 +402,12 @@ class TestSAEAutonomousRecovery:
"""Lock screen detected → SAE triggers unlock() → Instagram returns."""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
LOCK_SCREEN_XML, # perceive: locked
INSTAGRAM_HOME_XML, # verify after unlock
LOCK_SCREEN_XML, # perceive: locked
INSTAGRAM_HOME_XML, # verify after unlock
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.unlock.assert_called_once()
@@ -358,12 +420,11 @@ class TestSAEAutonomousRecovery:
INSTAGRAM_SURVEY_XML, # perceive: modal
INSTAGRAM_SURVEY_XML, # verify after BACK (BACK failed — modal still there)
INSTAGRAM_SURVEY_XML, # perceive again: still modal
INSTAGRAM_HOME_XML, # verify after clicking 'Not Now' (worked!)
INSTAGRAM_HOME_XML, # verify after clicking 'Not Now' (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# First action was BACK, second was click
@@ -378,30 +439,90 @@ class TestSAEAutonomousRecovery:
device = make_mock_device()
device.dump_hierarchy.side_effect = [
INSTAGRAM_SURVEY_XML, # perceive: modal
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.press.assert_called_with("back")
device.click.assert_not_called() # Never needed to click!
def test_recovers_from_unknown_modal_german(self):
"""German modal → BACK first → fails → finds 'Später' by TEXT → clicks."""
def test_recovers_from_randomized_chaos_modal(self, mock_telepathic_classifier, mock_fallback_llm):
"""Generates a totally random modal and verifies the LLM dictates the random coordinates to recover."""
import uuid
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"Nonsense_Text_{uuid.uuid4().hex[:8]}"
random_button_text = f"Dismiss_{uuid.uuid4().hex[:8]}"
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="{random_button_text}" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[321,2001][541,2101]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
UNKNOWN_MODAL_XML, # perceive: modal
UNKNOWN_MODAL_XML, # verify after BACK (failed)
UNKNOWN_MODAL_XML, # perceive again
chaos_xml, # perceive: modal
chaos_xml, # verify after BACK (failed)
chaos_xml, # perceive again
INSTAGRAM_HOME_XML, # verify after clicking randomized coords
]
# VLM Classifier override
def local_classifier(model, url, system_prompt, user_prompt, use_local_edge):
if random_text in user_prompt:
return '{"situation": "OBSTACLE_MODAL"}'
return '{"situation": "NORMAL"}'
mock_telepathic_classifier.side_effect = local_classifier
# VLM Fallback override (Action Solver)
def local_solver(*args, **kwargs):
prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "")
# Simulate real LLM: First it tries back, if 'back' is not in prompt
if "back:0,0" not in prompt.lower():
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Try back first"}'}
# Next time it sees the prompt, it finds the random button
if random_button_text in prompt:
# The bounds of our random button are [321,2001][541,2101] -> center is 431, 2051
return {"response": '{"action": "click", "x": 431, "y": 2051, "reason": "Found chaos button"}'}
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback"}'}
mock_fallback_llm.side_effect = local_solver
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# Proof that BACK was tried first
device.press.assert_called_with("back")
# Proof that the random coordinates were extracted and clicked
device.click.assert_called_once()
click_args = device.click.call_args
assert click_args[0] == (
431,
2051,
), f"Expected bot to click chaotic coordinates (431, 2051), but got {click_args[0]}"
def test_recovers_from_unknown_modal_german(self):
device = make_mock_device()
device.dump_hierarchy.side_effect = [
UNKNOWN_MODAL_XML, # perceive: modal
UNKNOWN_MODAL_XML, # verify after BACK (failed)
UNKNOWN_MODAL_XML, # perceive again
INSTAGRAM_HOME_XML, # verify after clicking 'Später'
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
device.click.assert_called_once()
@@ -409,7 +530,7 @@ class TestSAEAutonomousRecovery:
def test_never_clicks_close_friends_on_follow_sheet(self):
"""CRITICAL REAL-WORLD BUG: Follow sheet has 'close_friends' row.
SAE must NEVER click it — it adds the user to Close Friends!"""
follow_sheet_xml = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
follow_sheet_xml = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node package="com.instagram.android" bounds="[0,0][1080,2400]">
<node resource-id="com.instagram.android:id/bottom_sheet_container" package="com.instagram.android" bounds="[0,1400][1080,2400]">
@@ -418,16 +539,15 @@ class TestSAEAutonomousRecovery:
<node resource-id="com.instagram.android:id/follow_sheet_unfollow_row" text="Unfollow" package="com.instagram.android" clickable="true" bounds="[0,2193][1080,2335]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
follow_sheet_xml, # perceive: modal
follow_sheet_xml, # perceive: modal
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# CRITICAL: Must use BACK, never click any follow sheet button
@@ -449,12 +569,12 @@ class TestSAEAutonomousRecovery:
GOOGLE_SEARCH_XML, # attempt 5: perceive
GOOGLE_SEARCH_XML, # attempt 5: verify (LLM failed)
GOOGLE_SEARCH_XML, # attempt 6: perceive (escalate to app_start)
INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!)
INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!)
]
sae = SituationalAwarenessEngine(device)
# Mock LLM to return back action (simulating LLM also failing)
with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("back", reason="LLM says back")):
with patch.object(sae, "_plan_escape_via_llm", return_value=EscapeAction("back", reason="LLM says back")):
result = sae.ensure_clear_screen(max_attempts=7)
assert result is True
device.app_start.assert_called()
@@ -474,9 +594,10 @@ class TestSAEAutonomousRecovery:
def test_action_blocked_raises_exception(self):
"""If Instagram blocks us, SAE must HALT — never try to dismiss."""
from GramAddict.core.exceptions import ActionBlockedError
blocked_xml = INSTAGRAM_HOME_XML.replace(
'text="" resource-id="com.instagram.android:id/feed_tab"',
'text="Try again later" resource-id="com.instagram.android:id/dialog_container"'
'text="Try again later" resource-id="com.instagram.android:id/dialog_container"',
)
device = make_mock_device()
device.dump_hierarchy.return_value = blocked_xml
@@ -490,6 +611,7 @@ class TestSAEAutonomousRecovery:
# LEARNING TESTS
# ─────────────────────────────────────────────────────
class TestSAELearning:
"""Tests that SAE learns from experience and never repeats failures."""
@@ -536,15 +658,17 @@ class TestSAELearning:
"""When LLM returns 'false_positive', SAE must overwrite Qdrant and return True."""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
device.dump_hierarchy.return_value = INSTAGRAM_HOME_XML
# Force the situation to be perceived as an OBSTACLE_MODAL initially
with patch.object(sae, 'perceive', return_value=SituationType.OBSTACLE_MODAL):
with patch.object(sae, "perceive", return_value=SituationType.OBSTACLE_MODAL):
# Mock LLM to return 'false_positive'
with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("false_positive", reason="No modal found")):
with patch.object(
sae, "_plan_escape_via_llm", return_value=EscapeAction("false_positive", reason="No modal found")
):
result = sae.ensure_clear_screen(max_attempts=1, initial_xml=INSTAGRAM_HOME_XML)
assert result is True
mock_store_screen.assert_called_once()
args, kwargs = mock_store_screen.call_args

217
tests/e2e/test_full_e2e_android_sim.py Normal file
View File

@@ -0,0 +1,217 @@
import xml.etree.ElementTree as ET
from unittest.mock import MagicMock, patch
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.goap import GoalExecutor
from GramAddict.core.qdrant_memory import QdrantBase
from GramAddict.core.telepathic_engine import TelepathicEngine
class AndroidEnvironmentSimulator(DeviceFacade):
def __init__(self, device_id="sim", app_id="com.instagram.android", args=None):
self.device_id = device_id
self.app_id = app_id
self.args = args
self.deviceV2 = MagicMock()
self.deviceV2.info = {"displayWidth": 1080, "displayHeight": 2400, "screenOn": True}
self.state_stack = ["home_feed"]
self.state_files = {
"home_feed": "tests/fixtures/home_feed_with_ad.xml",
"explore_grid": "tests/fixtures/explore_feed_dump.xml",
"post_detail": "tests/fixtures/organic_post.xml",
"user_profile": "tests/fixtures/user_profile_dump.xml",
}
def _current_state(self):
return self.state_stack[-1]
def dump_hierarchy(self):
current = self._current_state()
filepath = self.state_files[current]
with open(filepath, "r", encoding="utf-8") as f:
data = f.read()
print(f"📱 [Simulator] dump_hierarchy returning state: {current} (length: {len(data)})")
return data
def _parse_bounds(self, bounds_str):
import re
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
if match:
return [int(x) for x in match.groups()]
return None
def human_click(self, x, y):
# Simulate click translation to next state
xml_data = self.dump_hierarchy()
root = ET.fromstring(xml_data)
clicked_nodes = []
for node in root.iter("node"):
bounds_str = node.attrib.get("bounds", "")
bounds = self._parse_bounds(bounds_str)
if bounds:
x1, y1, x2, y2 = bounds
if x1 <= x <= x2 and y1 <= y <= y2:
area = (x2 - x1) * (y2 - y1)
clicked_nodes.append((area, node))
if not clicked_nodes:
return
clicked_nodes.sort(key=lambda item: item[0])
for _, target in clicked_nodes:
content_desc = target.attrib.get("content-desc", "") or ""
res_id = target.attrib.get("resource-id", "") or ""
text = target.attrib.get("text", "") or ""
current = self._current_state()
if current == "home_feed":
if "Search and explore" in content_desc or "search_tab" in res_id:
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: home_feed -> explore_grid")
self.state_stack.append("explore_grid")
return
elif current == "explore_grid":
# In explore, anything the VLM clicks that has an image or button is likely a post
if "image_button" in res_id or "container" in res_id or target.attrib.get("clickable") == "true":
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: explore_grid -> post_detail")
self.state_stack.append("post_detail")
return
elif current == "post_detail":
# Allow clicking either the post author or the comment author (both go to user_profile)
if 100 < x < 800 and 300 < y < 900:
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: post_detail -> user_profile")
self.state_stack.append("user_profile")
return
# If we get here, no transition happened
for _, target in clicked_nodes:
print(
f"📱 [Simulator] Click ({x}, {y}) fell through on: {target.attrib.get('resource-id')} / text={target.attrib.get('text')}"
)
if not clicked_nodes:
print(f"📱 [Simulator] Click ({x}, {y}) fell outside ALL elements!")
def click(self, x=None, y=None, obj=None):
if x is not None and y is not None:
self.human_click(x, y)
elif obj and isinstance(obj, dict) and "x" in obj:
self.human_click(obj["x"], obj["y"])
def press(self, key):
if key == "back":
if len(self.state_stack) > 1:
old_state = self.state_stack.pop()
print(f"📱 [Simulator] Back pressed. State Transition: {old_state} -> {self._current_state()}")
else:
print("📱 [Simulator] Back pressed at root state.")
def _get_current_app(self):
return self.app_id
def get_info(self):
return self.deviceV2.info
def wake_up(self):
pass
def unlock(self):
pass
def shell(self, cmd):
return ""
def swipe(self, sx, sy, ex, ey, duration=None):
print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})")
def human_swipe(self, sx, sy, ex, ey, duration=None):
print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})")
@property
def info(self):
return self.deviceV2.info
@pytest.fixture(autouse=True)
def setup_qdrant_isolation():
"""Prefix all Qdrant collections with test_sim_ so we don't pollute live data."""
original_init = QdrantBase.__init__
def mocked_init(self, collection_name, *args, **kwargs):
test_collection = f"test_sim_{collection_name}"
original_init(self, test_collection, *args, **kwargs)
with patch.object(QdrantBase, "__init__", new=mocked_init):
# We aggressively wipe these collections before running the test!
from GramAddict.core.qdrant_memory import NavigationMemoryDB
qb = NavigationMemoryDB()
try:
qb.wipe_collection()
except:
pass
yield
def test_full_autonomous_sim_loop(monkeypatch):
"""
This test runs the real GoalExecutor with the real TelepathicEngine (VLM)
and real Qdrant (sandboxed via prefix) against a simulated Android environment.
"""
import urllib.request
try:
urllib.request.urlopen("http://localhost:11434/", timeout=2)
except Exception:
pytest.skip("Ollama is not running. Live E2E sim requires LLM backend.")
# 1. Create Simulator
sim_device = AndroidEnvironmentSimulator()
# 2. Patch TelepathicEngine to NOT be mocked by conftest
engine = TelepathicEngine()
monkeypatch.setattr(TelepathicEngine, "get_instance", lambda: engine)
# 3. Create context and GoalExecutor
from GramAddict.core.config import Config
if not hasattr(Config(), "args"):
Config().args = MagicMock()
Config().args.use_nav_memory = True
Config().args.use_semantic_memory = True
executor = GoalExecutor(sim_device, bot_username="testbot")
# 4. Start an autonomous loop: We want to reach an organic post from the home feed
assert sim_device._current_state() == "home_feed"
success = executor.achieve("open post", max_steps=10)
assert success is True
# The VLM should have figured out:
# 1. Tap explore tab -> switches to "explore_grid"
# 2. Tap grid item -> switches to "post_detail"
assert sim_device._current_state() == "post_detail"
# 5. Let's do another intent: view the user profile
success = executor.achieve("open post author profile", max_steps=5)
assert success is True
assert sim_device._current_state() == "user_profile"
# 6. Now go back to the post
success = executor.achieve("open post", max_steps=5)
assert success is True
assert sim_device._current_state() == "post_detail"
# 7. Check Qdrant Memory is actually populated
# We should have stored the state transitions in the goap_paths collection
from GramAddict.core.goap import PathMemory
nav_db = PathMemory("testbot")
# verify at least some nodes exist
count = nav_db._db.client.count(nav_db._db.collection_name).count
assert count > 0, "Qdrant memory should have learned the paths!"

470
tests/integration/test_bot_flow_interaction.py
View File

@@ -1,14 +1,15 @@
import pytest
from unittest.mock import MagicMock, patch
import pytest
from GramAddict.core.bot_flow import (
_align_active_post,
_extract_post_content,
_run_zero_latency_feed_loop,
_run_zero_latency_stories_loop,
_extract_post_content,
is_ad,
_align_active_post
)
from GramAddict.core.session_state import SessionState
@pytest.fixture
def mock_device():
@@ -20,12 +21,12 @@ def mock_device():
return device
@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance')
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
def test_extract_post_content(mock_get_telepathic):
mock_engine = MagicMock()
mock_engine.find_best_node.side_effect = [
{"original_attribs": {"text": "test_user"}},
{"original_attribs": {"desc": "test description of image with more than 10 chars"}}
{"original_attribs": {"desc": "test description of image with more than 10 chars"}},
]
mock_get_telepathic.return_value = mock_engine
xml = "<xml/>"
@@ -33,19 +34,21 @@ def test_extract_post_content(mock_get_telepathic):
assert res["username"] == "test_user"
assert "test description" in res["description"]
@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance')
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
def test_extract_post_content_fallback_caption(mock_get_telepathic):
mock_engine = MagicMock()
mock_engine.find_best_node.side_effect = [{"original_attribs": {"text": "other_user"}}, None]
mock_get_telepathic.return_value = mock_engine
xml = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
xml = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy>
<node resource-id="" text="other_user this is a very long caption text that we want to extract as fallback" />
</hierarchy>'''
</hierarchy>"""
res = _extract_post_content(xml)
assert res["username"] == "other_user"
assert "this is a very long caption" in res["caption"]
def testis_ad():
assert is_ad('<node resource-id="com.instagram.android:id/ad_cta_button" />') == True
assert is_ad('<node resource-id="com.instagram.android:id/clips_single_image_ads_media_content" />') == True
@@ -53,7 +56,8 @@ def testis_ad():
assert is_ad('<node resource-id="com.instagram.android:id/secondary_label" text="regular post" />') == False
assert is_ad('<node resource-id="com.instagram.android:id/normal_post" />') == False
@patch('GramAddict.core.telepathic_engine.TelepathicEngine.get_instance')
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
def test_align_active_post(mock_get_telepathic, mock_device):
# Test snapping when post is far from ideal coordinates
mock_engine = MagicMock()
@@ -64,127 +68,176 @@ def test_align_active_post(mock_get_telepathic, mock_device):
# The header is at 850px. Target is 250px. Diff is 600px. It should swipe.
assert mock_device.swipe.called
def test_feed_loop_boredom_change_feed(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False
mock_cognitive_stack["growth_brain"].evaluate_governance.return_value = "SHIFT_CONTEXT"
configs = MagicMock()
session_state = MagicMock()
session_state.check_limit.return_value = (False, False, False, False)
res = _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
res = _run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert res == "BOREDOM_CHANGE_FEED"
def test_feed_loop_context_lost(mock_device, mock_cognitive_stack):
# Simulate not having any feed markers 3 times
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, False, False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_device.dump_hierarchy.return_value = "<hierarchy></hierarchy>" # Blind
mock_device.dump_hierarchy.return_value = "<hierarchy></hierarchy>" # Blind
# Needs telepathic engine mock
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, patch('GramAddict.core.bot_flow.dump_ui_state'):
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow.dump_ui_state"),
):
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}] # Pretend we have nodes so it doesn't trigger zero-node immediately
mock_instance._extract_semantic_nodes.return_value = [
{"x": 1, "y": 2}
] # Pretend we have nodes so it doesn't trigger zero-node immediately
configs = MagicMock()
session_state = MagicMock()
session_state.check_limit.return_value = (False, False, False, False)
res = _run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
res = _run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert res == "CONTEXT_LOST"
def test_feed_loop_zero_nodes(mock_device, mock_cognitive_stack):
# Tests the Zero-Node recovery anomaly handler
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll:
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll,
):
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [] # Zero interactive nodes
mock_instance._extract_semantic_nodes.return_value = [] # Zero interactive nodes
configs = MagicMock()
session_state = MagicMock()
session_state.check_limit.return_value = (False, False, False, False)
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert mock_device.press.called_with("back")
assert mock_scroll.called
def test_feed_loop_ad_skip(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_device.dump_hierarchy.return_value = '''<?xml version='1.0' ?>
mock_device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_button_like" />
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="ad_account" />
<node resource-id="com.instagram.android:id/ad_cta_button" />
</hierarchy>'''
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \
patch('GramAddict.core.bot_flow._align_active_post') as mock_align:
</hierarchy>"""
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll,
patch("GramAddict.core.bot_flow._align_active_post") as mock_align,
):
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [{"x": 1}]
mock_align.return_value = False
configs = MagicMock()
session_state = MagicMock()
session_state.check_limit.return_value = (False, False, False, False)
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert mock_scroll.called
def test_stories_loop_success(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
configs = MagicMock()
configs.args.stories = "1"
session_state = MagicMock()
mock_device.dump_hierarchy.return_value = '''<?xml version='1.0' ?>
mock_device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/story_viewer" />
</hierarchy>'''
with patch('GramAddict.core.bot_flow._humanized_click') as mock_click, patch('GramAddict.core.bot_flow.sleep'):
</hierarchy>"""
with patch("GramAddict.core.bot_flow._humanized_click") as mock_click, patch("GramAddict.core.bot_flow.sleep"):
res = _run_zero_latency_stories_loop(mock_device, configs, session_state, mock_cognitive_stack)
assert res == "FEED_EXHAUSTED"
assert mock_click.called
def test_stories_loop_boredom(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = True
configs = MagicMock()
session_state = MagicMock()
with patch('GramAddict.core.bot_flow.sleep'):
with patch("GramAddict.core.bot_flow.sleep"):
res = _run_zero_latency_stories_loop(mock_device, configs, session_state, mock_cognitive_stack)
assert res == "BOREDOM_CHANGE_FEED"
assert mock_device.press.called_with("back")
def test_start_bot_interrupt():
from GramAddict.core.bot_flow import start_bot
# Mock all the heavy initialization
with patch('GramAddict.core.bot_flow.Config') as MockConfig, \
patch('GramAddict.core.bot_flow.configure_logger'), \
patch('GramAddict.core.bot_flow.check_if_updated'), \
patch('GramAddict.core.benchmark_guard.check_model_benchmarks'), \
patch('GramAddict.core.llm_provider.log_openrouter_burn'), \
patch('GramAddict.core.bot_flow.create_device') as mock_create_device, \
patch('GramAddict.core.bot_flow.set_time_delta') as mock_time_delta, \
patch('GramAddict.core.bot_flow.SessionState') as MockSession, \
patch('GramAddict.core.bot_flow.open_instagram', side_effect=KeyboardInterrupt()):
# Mock all the heavy initialization
with (
patch("GramAddict.core.bot_flow.Config") as MockConfig,
patch("GramAddict.core.bot_flow.configure_logger"),
patch("GramAddict.core.bot_flow.check_if_updated"),
patch("GramAddict.core.benchmark_guard.check_model_benchmarks"),
patch("GramAddict.core.llm_provider.log_openrouter_burn"),
patch("GramAddict.core.bot_flow.create_device") as mock_create_device,
patch("GramAddict.core.bot_flow.set_time_delta") as mock_time_delta,
patch("GramAddict.core.bot_flow.SessionState") as MockSession,
patch("GramAddict.core.bot_flow.open_instagram", side_effect=KeyboardInterrupt()),
):
MockConfig.return_value.args.feed = True
MockConfig.return_value.args.explore = False
MockConfig.return_value.args.reels = False
@@ -197,19 +250,20 @@ def test_start_bot_interrupt():
MockConfig.return_value.args.ai_embedding_url = "http://localhost:11434/api/chat"
MockConfig.return_value.args.ai_embedding_model = "llama3"
MockConfig.return_value.args.agent_strategy = "conservative"
MockSession.inside_working_hours.return_value = (True, 0)
with pytest.raises(KeyboardInterrupt):
start_bot(username="test_user", device_id="123")
def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack):
# This test hits the core interaction (Lines 900 - 1300)
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.85
configs = MagicMock()
configs.args.likes_percentage = 100
configs.args.follow_percentage = 100
@@ -218,13 +272,15 @@ def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack):
configs.args.ai_condenser_model = "test-model"
configs.args.ai_condenser_url = "test-url"
configs.args.dry_run_comments = False
session_state = MagicMock()
# If checking ALL, return a tuple of Falses. If specific limit like LIKES/COMMENTS, return False bool.
session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
session_state.check_limit.side_effect = (
lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
)
# Needs to report a structure that has NO ad, HAS content, and HAS feed markers.
mock_device.dump_hierarchy.return_value = '''<?xml version='1.0' ?>
mock_device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_button_like" />
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="legit_user" />
@@ -234,142 +290,182 @@ def test_feed_loop_deep_engagement(mock_device, mock_cognitive_stack):
<node resource-id="com.instagram.android:id/row_comment_textview_comment" text="This is a fantastic picture!" />
<node resource-id="com.instagram.android:id/row_comment_button_like" bounds="[10,10][20,20]" />
</node>
</hierarchy>'''
</hierarchy>"""
# Ensure radome doesn't destroy our XML string
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
# Simulate that nav_graph transitions work
mock_cognitive_stack["nav_graph"].do.return_value = True
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \
patch('GramAddict.core.llm_provider.query_llm') as mock_llm, \
patch('GramAddict.core.bot_flow.random.random', return_value=0.11), \
patch('GramAddict.core.bot_flow.random.uniform', return_value=1.5), \
patch('GramAddict.core.bot_flow.random.randint', return_value=1), \
patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \
patch('GramAddict.core.bot_flow._humanized_scroll') as mock_scroll, \
patch('GramAddict.core.bot_flow._humanized_click') as mock_click, \
patch('GramAddict.core.stealth_typing.ghost_type') as mock_type:
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract,
patch("GramAddict.core.llm_provider.query_llm") as mock_llm,
patch("GramAddict.core.bot_flow.random.random", return_value=0.11),
patch("GramAddict.core.bot_flow.random.uniform", return_value=1.5),
patch("GramAddict.core.bot_flow.random.randint", return_value=1),
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll,
patch("GramAddict.core.bot_flow._humanized_click") as mock_click,
patch("GramAddict.core.stealth_typing.ghost_type") as mock_type,
):
mock_extract.return_value = {"username": "legit_user", "description": "test image", "caption": ""}
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2, "original_attribs": {"text": "This is a fantastic picture!"}}]
mock_instance._extract_semantic_nodes.return_value = [
{"x": 1, "y": 2, "original_attribs": {"text": "This is a fantastic picture!"}}
]
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_llm.return_value = {"response": "Great shot!"}
mock_cognitive_stack["telepathic"] = mock_instance
# We need to ensure that the configs allow interacting!
configs.args.interact_percentage = 100
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert mock_click.called
assert mock_type.called
def test_feed_loop_repost(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.85
configs = MagicMock()
configs.args.repost_percentage = 100
configs.args.likes_percentage = 0
configs.args.comment_percentage = 0
session_state = MagicMock()
session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
mock_device.dump_hierarchy.return_value = '''<?xml version='1.0' ?>
session_state.check_limit.side_effect = (
lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
)
mock_device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_button_like" />
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="legit_user" />
<node resource-id="com.instagram.android:id/row_feed_photo_imageview" content-desc="test image" />
</hierarchy>'''
</hierarchy>"""
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
mock_cognitive_stack["nav_graph"].do.return_value = True
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow.random.random', return_value=0.11), \
patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \
patch('GramAddict.core.bot_flow._humanized_scroll'), \
patch('GramAddict.core.bot_flow._humanized_click') as mock_click:
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow.random.random", return_value=0.11),
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll"),
patch("GramAddict.core.bot_flow._humanized_click") as mock_click,
):
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}]
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_cognitive_stack["telepathic"] = mock_instance
configs.args.interact_percentage = 100
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
def test_profile_learning_percentage_trigger(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 # Not high enough to trigger default
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50 # Not high enough to trigger default
configs = MagicMock()
configs.args.profile_learning_percentage = 100 # Should force visit
configs.args.profile_learning_percentage = 100 # Should force visit
configs.args.likes_percentage = 0
configs.args.comment_percentage = 0
configs.args.follow_percentage = 0 # Won't trigger by follow chance either
configs.args.follow_percentage = 0 # Won't trigger by follow chance either
session_state = MagicMock()
session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
mock_device.dump_hierarchy.return_value = '''<?xml version='1.0' ?>
session_state.check_limit.side_effect = (
lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
)
mock_device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="legit_user" />
<node resource-id="com.instagram.android:id/row_feed_photo_imageview" content-desc="test image" />
</hierarchy>'''
</hierarchy>"""
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
mock_cognitive_stack["nav_graph"].do.return_value = True
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \
patch('GramAddict.core.bot_flow.random.random', return_value=0.5), \
patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \
patch('GramAddict.core.bot_flow._humanized_scroll'), \
patch('GramAddict.core.bot_flow._interact_with_profile') as mock_interact:
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract,
patch("GramAddict.core.bot_flow.random.random", return_value=0.5),
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll"),
patch("GramAddict.core.bot_flow._interact_with_profile") as mock_interact,
):
mock_extract.return_value = {"username": "legit_user", "description": "test image", "caption": ""}
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2, "original_attribs": {"text": "dummy"}}]
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_cognitive_stack["telepathic"] = mock_instance
configs.args.interact_percentage = 100
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert mock_interact.called
def test_ai_learn_own_profile_triggers_goap():
with patch('GramAddict.core.bot_flow.Config') as MockConfig, \
patch('GramAddict.core.bot_flow.configure_logger'), \
patch('GramAddict.core.bot_flow.check_if_updated'), \
patch('GramAddict.core.benchmark_guard.check_model_benchmarks'), \
patch('GramAddict.core.llm_provider.log_openrouter_burn'), \
patch('GramAddict.core.llm_provider.prewarm_ollama_models'), \
patch('GramAddict.core.bot_flow.create_device') as mock_create_device, \
patch('GramAddict.core.bot_flow.set_time_delta'), \
patch('GramAddict.core.bot_flow.SessionState') as MockSession, \
patch('GramAddict.core.bot_flow.open_instagram', return_value=True), \
patch('GramAddict.core.bot_flow.verify_and_switch_account', return_value=True), \
patch('GramAddict.core.bot_flow.get_instagram_version', return_value="1.0"), \
patch('GramAddict.core.goap.GoalExecutor') as MockGoalExecutor, \
patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.llm_provider.query_llm') as mock_query, \
patch('GramAddict.core.bot_flow.DojoEngine'), \
patch('GramAddict.core.bot_flow.sleep'):
with (
patch("GramAddict.core.bot_flow.Config") as MockConfig,
patch("GramAddict.core.bot_flow.configure_logger"),
patch("GramAddict.core.bot_flow.check_if_updated"),
patch("GramAddict.core.benchmark_guard.check_model_benchmarks"),
patch("GramAddict.core.llm_provider.log_openrouter_burn"),
patch("GramAddict.core.llm_provider.prewarm_ollama_models"),
patch("GramAddict.core.bot_flow.create_device") as mock_create_device,
patch("GramAddict.core.bot_flow.set_time_delta"),
patch("GramAddict.core.bot_flow.SessionState") as MockSession,
patch("GramAddict.core.bot_flow.open_instagram", return_value=True),
patch("GramAddict.core.bot_flow.verify_and_switch_account", return_value=True),
patch("GramAddict.core.bot_flow.get_instagram_version", return_value="1.0"),
patch("GramAddict.core.goap.GoalExecutor") as MockGoalExecutor,
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.llm_provider.query_llm") as mock_query,
patch("GramAddict.core.bot_flow.DojoEngine"),
patch("GramAddict.core.bot_flow.sleep"),
):
MockConfig.return_value.args.ai_learn_own_profile = True
MockConfig.return_value.args.agent_strategy = "aggressive_growth"
MockConfig.return_value.args.capture_e2e_dumps = False
@@ -379,26 +475,25 @@ def test_ai_learn_own_profile_triggers_goap():
MockConfig.return_value.args.stories = False
MockConfig.return_value.args.working_hours = [10, 20]
MockConfig.return_value.args.time_delta_session = 30
MockSession.inside_working_hours.return_value = (True, 0)
mock_goap = MockGoalExecutor.get_instance.return_value
mock_goap.achieve.return_value = True
mock_telepathic = MockTelepathic.return_value # the class constructor is called inside start_bot
mock_telepathic._extract_semantic_nodes.return_value = [
{"original_attribs": {"text": "my cool bio"}}
]
mock_telepathic = MockTelepathic.return_value # the class constructor is called inside start_bot
mock_telepathic._extract_semantic_nodes.return_value = [{"original_attribs": {"text": "my cool bio"}}]
mock_query.return_value = {"persona": "cool dev", "vibe": "chill"}
from GramAddict.core.bot_flow import start_bot
try:
with patch('GramAddict.core.bot_flow.random_sleep', side_effect=KeyboardInterrupt()):
with patch("GramAddict.core.bot_flow.random_sleep", side_effect=KeyboardInterrupt()):
start_bot(username="testuser", device_id="123")
except KeyboardInterrupt:
pass
mock_goap.achieve.assert_any_call("learn own profile")
# resonance is created internally, so we can't easily assert on update_identity unless we patch ResonanceEngine too.
# It's sufficient to know the GOAP goal was triggered.
@@ -408,58 +503,75 @@ def test_profile_mismatch_recovery(mock_device, mock_cognitive_stack):
mock_cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50
mock_cognitive_stack["resonance"].calculate_resonance.return_value = 0.50
configs = MagicMock()
configs.args.profile_learning_percentage = 100 # Should force visit
configs.args.profile_learning_percentage = 100 # Should force visit
configs.args.likes_percentage = 0
configs.args.comment_percentage = 0
configs.args.follow_percentage = 0
configs.args.follow_percentage = 0
session_state = MagicMock()
session_state.check_limit.side_effect = lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
feed_xml = '''<?xml version='1.0' ?>
session_state.check_limit.side_effect = (
lambda limit_type: (False, False, False, False) if getattr(limit_type, "name", "") == "ALL" else False
)
feed_xml = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="amorextravel" />
<node resource-id="com.instagram.android:id/row_feed_photo_imageview" content-desc="test image" />
</hierarchy>'''
profile_xml = '''<?xml version='1.0' ?>
</hierarchy>"""
profile_xml = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/action_bar_title" text="ryanresatka" />
<node resource-id="com.instagram.android:id/row_profile_header_textview_biography" text="cool bio" />
</hierarchy>'''
</hierarchy>"""
call_count = [0]
def dump_hierarchy_mock(*args, **kwargs):
call_count[0] += 1
return feed_xml if call_count[0] == 1 else profile_xml
mock_device.dump_hierarchy.side_effect = dump_hierarchy_mock
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
mock_cognitive_stack["nav_graph"].do.return_value = True
with patch('GramAddict.core.bot_flow.TelepathicEngine') as MockTelepathic, \
patch('GramAddict.core.bot_flow._extract_post_content') as mock_extract, \
patch('GramAddict.core.bot_flow.random.random', return_value=0.5), \
patch('GramAddict.core.bot_flow._align_active_post', return_value=False), \
patch('GramAddict.core.bot_flow._humanized_scroll'), \
patch('GramAddict.core.bot_flow._interact_with_profile') as mock_interact:
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract,
patch("GramAddict.core.bot_flow.random.random", return_value=0.5),
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll"),
patch("GramAddict.core.bot_flow._interact_with_profile") as mock_interact,
):
mock_extract.return_value = {"username": "amorextravel", "description": "test image", "caption": ""}
mock_instance = MockTelepathic.get_instance.return_value
mock_instance._extract_semantic_nodes.side_effect = [
[{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 1st call at top of loop
[{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 2nd call before Targeted UX
[{"x": 1, "y": 2, "text": "ryanresatka", "resource_id": "com.instagram.android:id/action_bar_title"}] # 3rd call on profile
[{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 1st call at top of loop
[{"x": 1, "y": 2, "original_attribs": {"text": "amorextravel"}}], # 2nd call before Targeted UX
[
{"x": 1, "y": 2, "text": "ryanresatka", "resource_id": "com.instagram.android:id/action_bar_title"}
], # 3rd call on profile
]
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_cognitive_stack["telepathic"] = mock_instance
configs.args.interact_percentage = 100
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
_run_zero_latency_feed_loop(mock_device, mock_cognitive_stack["zero_engine"], mock_cognitive_stack["nav_graph"], configs, session_state, "HomeFeed", mock_cognitive_stack)
assert mock_interact.call_args[0][2] == "ryanresatka", f"Expected ryanresatka but got {mock_interact.call_args[0][2]}"
_run_zero_latency_feed_loop(
mock_device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
assert (
mock_interact.call_args[0][2] == "ryanresatka"
), f"Expected ryanresatka but got {mock_interact.call_args[0][2]}"

39
tests/unit/perception/test_action_memory.py Normal file
View File

@@ -0,0 +1,39 @@
from unittest.mock import patch
import pytest
from GramAddict.core.perception.action_memory import ActionMemory
from GramAddict.core.perception.spatial_parser import SpatialNode
@pytest.fixture
def memory():
with patch("GramAddict.core.qdrant_memory.UIMemoryDB") as MockDB:
mock_db = MockDB.return_value
yield ActionMemory(ui_memory=mock_db)
def test_track_click_stores_memory(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
assert memory._last_click_context is not None
assert memory._last_click_context["intent"] == "tap test"
def test_confirm_click_boosts_confidence(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
memory.confirm_click()
memory.ui_memory.boost_confidence.assert_called_once()
assert memory._last_click_context is None
def test_reject_click_decays_confidence(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
memory.reject_click()
memory.ui_memory.decay_confidence.assert_called_once()
assert memory._last_click_context is None

37
tests/unit/perception/test_intent_resolver.py Normal file
View File

@@ -0,0 +1,37 @@
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialNode
def test_intent_resolver_finds_bottom_tab():
resolver = IntentResolver()
# A tab at the top
top_tab = SpatialNode(bounds=(0, 0, 100, 100), content_desc="Explore Tab", clickable=True)
# A tab at the bottom
bottom_tab = SpatialNode(bounds=(0, 2200, 100, 2300), content_desc="Explore Tab", clickable=True)
# Intent resolver should prefer the one that geometrically matches the bottom navigation area
best_match = resolver.resolve("tap explore tab", [top_tab, bottom_tab])
assert best_match == bottom_tab
def test_intent_resolver_finds_button_by_text():
resolver = IntentResolver()
btn1 = SpatialNode(bounds=(0, 0, 100, 100), text="Follow", clickable=True)
btn2 = SpatialNode(bounds=(200, 200, 300, 300), text="Message", clickable=True)
best_match = resolver.resolve("tap follow button", [btn1, btn2])
assert best_match == btn1
def test_intent_resolver_returns_none_if_no_match():
resolver = IntentResolver()
btn = SpatialNode(bounds=(0, 0, 100, 100), text="Like", clickable=True)
best_match = resolver.resolve("tap follow button", [btn])
assert best_match is None

77
tests/unit/perception/test_spatial_parser.py Normal file
View File

@@ -0,0 +1,77 @@
from GramAddict.core.perception.spatial_parser import SpatialNode, SpatialParser
XML_FIXTURE = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" bounds="[0,0][1080,2400]" class="android.widget.FrameLayout">
<node index="0" bounds="[0,2200][1080,2400]" class="android.view.ViewGroup" content-desc="Bottom Navigation">
<node index="0" bounds="[50,2250][150,2350]" class="android.widget.ImageView" content-desc="Home Tab" clickable="true"/>
<node index="1" bounds="[250,2250][350,2350]" class="android.widget.ImageView" content-desc="Explore Tab" clickable="true"/>
</node>
<node index="1" bounds="[0,100][1080,2200]" class="androidx.recyclerview.widget.RecyclerView">
<node index="0" bounds="[50,150][1030,800]" class="android.widget.FrameLayout" content-desc="Post 1">
<node index="0" bounds="[100,700][200,750]" class="android.widget.Button" text="Like" clickable="true"/>
</node>
</node>
</node>
</hierarchy>
"""
class TestSpatialParser:
def test_parses_xml_into_spatial_nodes(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
assert root is not None
assert isinstance(root, SpatialNode)
assert root.bounds == (0, 0, 1080, 2400)
assert len(root.children) == 2 # The Bottom Nav and the Recycler View
def test_extracts_all_clickable_nodes(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
clickables = parser.get_clickable_nodes(root)
assert len(clickables) == 3
descriptions = [n.content_desc or n.text for n in clickables]
assert "Home Tab" in descriptions
assert "Explore Tab" in descriptions
assert "Like" in descriptions
def test_spatial_containment(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
# Get the first post
post_nodes = [n for n in parser.get_all_nodes(root) if n.content_desc == "Post 1"]
assert len(post_nodes) == 1
post = post_nodes[0]
# Get the Like button
like_nodes = [n for n in parser.get_all_nodes(root) if n.text == "Like"]
assert len(like_nodes) == 1
like_btn = like_nodes[0]
# Spatial Containment: The Like button should be mathematically inside the Post
assert post.contains(like_btn) is True
# The Like button should NOT contain the Post
assert like_btn.contains(post) is False
# The Home Tab should NOT be in the Post
home_tabs = [n for n in parser.get_all_nodes(root) if n.content_desc == "Home Tab"]
assert post.contains(home_tabs[0]) is False
def test_spatial_intersection(self):
parser = SpatialParser()
# Node 1: Left side
n1 = SpatialNode(bounds=(0, 0, 100, 100))
# Node 2: Overlapping Right side
n2 = SpatialNode(bounds=(50, 50, 150, 150))
# Node 3: Far away
n3 = SpatialNode(bounds=(200, 200, 300, 300))
assert n1.intersects(n2) is True
assert n2.intersects(n1) is True
assert n1.intersects(n3) is False

65
tests/unit/test_bot_flow_unlearn.py Normal file
View File

@@ -0,0 +1,65 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
def test_bot_flow_unlearns_on_context_loss():
"""Prove that bot_flow calls unlearn_current_state when context is completely lost (3 misses)."""
device = MagicMock()
# Provide a dummy dump hierarchy
device.dump_hierarchy.return_value = "<hierarchy></hierarchy>"
device.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400}
session_state = MagicMock()
# We will patch SituationalAwarenessEngine
with patch("GramAddict.core.situational_awareness.SituationalAwarenessEngine") as MockSAE:
# We need mock SAE to return OBSTACLE_MODAL to trigger the first condition
# Wait, the code has two paths: `has_obstacle` or `not has_feed_markers`.
# If we return `False` for has_feed_markers, it hits the second path.
mock_sae_instance = MockSAE.return_value
# perceive needs to return something that is not OBSTACLE_MODAL so we hit the feed markers path
mock_sae_instance.perceive.return_value = "EXPLORE_GRID"
# Act: _run_zero_latency_feed_loop runs a loop.
# Since has_feed_markers is always False, it will increment misses 3 times and return "CONTEXT_LOST".
# We also need to mock TelepathicEngine so it doesn't crash on misses == 2.
with (
patch("GramAddict.core.bot_flow.TelepathicEngine") as MockTelepathic,
patch("GramAddict.core.bot_flow.dump_ui_state"),
):
mock_telepathic_instance = MockTelepathic.get_instance.return_value
mock_telepathic_instance.find_best_node.return_value = None
mock_telepathic_instance._extract_semantic_nodes.return_value = [MagicMock()]
mock_cognitive_stack = MagicMock()
dopamine_mock = MagicMock()
dopamine_mock.is_app_session_over.return_value = False
dopamine_mock.wants_to_doomscroll.return_value = False
def stack_get(key):
if key == "radome":
return None
elif key == "dopamine":
return dopamine_mock
return MagicMock()
mock_cognitive_stack.get.side_effect = stack_get
with patch("GramAddict.core.bot_flow.is_ad", return_value=False):
result = _run_zero_latency_feed_loop(
device=device,
zero_engine=MagicMock(),
nav_graph=MagicMock(),
configs=MagicMock(),
session_state=session_state,
job_target="test_feed",
cognitive_stack=mock_cognitive_stack,
)
# Assert (RED)
assert result == "CONTEXT_LOST"
# SAE should have been told to unlearn the current state because of context loss
mock_sae_instance.unlearn_current_state.assert_called_with("<hierarchy></hierarchy>")

169
tests/unit/test_explore_grid_navigation.py
View File

@@ -8,67 +8,91 @@ Reproduces the exact production failure from 2026-04-16 22:59 where the bot:
These tests MUST fail before the fix and pass after.
"""
import pytest
import re
from GramAddict.core.telepathic_engine import TelepathicEngine
import re
from GramAddict.core.telepathic_engine import TelepathicEngine
# ── Realistic node fixtures extracted from live XML dump 2026-04-16_22-59-53 ──
def make_node(x, y, bounds, semantic, res_id="com.instagram.android:id/dummy", text="", desc=""):
m = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
l, t, r, b = map(int, m.groups()) if m else (0, 0, 0, 0)
return {
"x": x, "y": y,
"width": r - l, "height": b - t, "area": (r - l) * (b - t),
"x": x,
"y": y,
"width": r - l,
"height": b - t,
"area": (r - l) * (b - t),
"raw_bounds": bounds,
"semantic_string": semantic,
"resource_id": res_id,
"class_name": "android.widget.FrameLayout",
"selected": False,
"original_attribs": {"text": text, "desc": desc}
"original_attribs": {"text": text, "desc": desc},
}
EXPLORE_GRID_NODES = [
# Row 1, Col 1 — this is what "first image" should match
make_node(178, 559, "[0,321][356,797]",
"description: '4 photos by Patsy Weingart at row 1, column 1', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="4 photos by Patsy Weingart at row 1, column 1"),
make_node(
178,
559,
"[0,321][356,797]",
"description: '4 photos by Patsy Weingart at row 1, column 1', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="4 photos by Patsy Weingart at row 1, column 1",
),
# Row 1, Col 1 — child image_button (same area, no semantic info)
make_node(178, 558, "[0,321][356,796]",
"id context: 'image button'",
res_id="com.instagram.android:id/image_button"),
make_node(
178, 558, "[0,321][356,796]", "id context: 'image button'", res_id="com.instagram.android:id/image_button"
),
# Row 1, Col 2
make_node(540, 559, "[362,321][718,797]",
"description: 'Photo by Barbara at Row 1, Column 2', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="Photo by Barbara at Row 1, Column 2"),
make_node(540, 558, "[362,321][718,796]",
"id context: 'image button'",
res_id="com.instagram.android:id/image_button"),
make_node(
540,
559,
"[362,321][718,797]",
"description: 'Photo by Barbara at Row 1, Column 2', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="Photo by Barbara at Row 1, Column 2",
),
make_node(
540, 558, "[362,321][718,796]", "id context: 'image button'", res_id="com.instagram.android:id/image_button"
),
# Row 2, Col 2
make_node(540, 1041, "[362,803][718,1279]",
"description: 'Photo by Garima Bhaskar at Row 2, Column 2', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="Photo by Garima Bhaskar at Row 2, Column 2"),
make_node(540, 1040, "[362,803][718,1278]",
"id context: 'image button'",
res_id="com.instagram.android:id/image_button"),
make_node(
540,
1041,
"[362,803][718,1279]",
"description: 'Photo by Garima Bhaskar at Row 2, Column 2', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="Photo by Garima Bhaskar at Row 2, Column 2",
),
make_node(
540, 1040, "[362,803][718,1278]", "id context: 'image button'", res_id="com.instagram.android:id/image_button"
),
# Row 3, Col 1 — this is what the VLM wrongly picked
make_node(178, 1523, "[0,1285][356,1761]",
"description: '4 photos by Soul Of Nature Photography at row 3, column 1', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="4 photos by Soul Of Nature Photography at row 3, column 1"),
make_node(178, 1522, "[0,1285][356,1760]",
"id context: 'image button'",
res_id="com.instagram.android:id/image_button"),
make_node(
178,
1523,
"[0,1285][356,1761]",
"description: '4 photos by Soul Of Nature Photography at row 3, column 1', id context: 'grid card layout container'",
res_id="com.instagram.android:id/grid_card_layout_container",
desc="4 photos by Soul Of Nature Photography at row 3, column 1",
),
make_node(
178, 1522, "[0,1285][356,1760]", "id context: 'image button'", res_id="com.instagram.android:id/image_button"
),
# Search bar
make_node(487, 219, "[32,173][943,265]",
"text: 'Search', id context: 'action bar search edit text'",
res_id="com.instagram.android:id/action_bar_search_edit_text",
text="Search"),
make_node(
487,
219,
"[32,173][943,265]",
"text: 'Search', id context: 'action bar search edit text'",
res_id="com.instagram.android:id/action_bar_search_edit_text",
text="Search",
),
]
@@ -87,17 +111,18 @@ class TestBlacklistPoisoning:
engine = TelepathicEngine()
# Clear any persisted state to test pure logic
engine._blacklist = {}
# Simulate the flow: the engine "clicked" an image_button and it failed
TelepathicEngine._last_click_context = {
"intent": "first image in explore grid",
"semantic_string": "id context: 'image button'",
"x": 178, "y": 558,
"timestamp": 1000
"x": 178,
"y": 558,
"timestamp": 1000,
}
engine.reject_click("first image in explore grid")
# The blacklist should NOT contain this generic entry
blacklisted = engine._blacklist.get("first image in explore grid", [])
assert "id context: 'image button'" not in blacklisted, (
@@ -123,18 +148,20 @@ class TestExploreGridFastPath:
# Build a minimal XML that the engine can parse — but we test the fast-path
# directly by calling find_best_node with mocked extraction
from unittest.mock import patch
with patch.object(engine, '_extract_semantic_nodes', return_value=EXPLORE_GRID_NODES):
with patch.object(engine, '_is_instagram_context', return_value=True):
result = engine.find_best_node("<fake>", "first image in explore grid", min_confidence=0.82, device=None)
with patch.object(engine, "_extract_semantic_nodes", return_value=EXPLORE_GRID_NODES):
with patch.object(engine, "_is_instagram_context", return_value=True):
result = engine.find_best_node(
"<fake>", "first image in explore grid", min_confidence=0.82, device=None
)
assert result is not None, (
"Grid Fast-Path returned None for 'first image in explore grid'. "
"This forces every explore grid tap to use the expensive VLM fallback."
)
assert any(k in result.get("semantic", "").lower() for k in ["image button", "grid card layout container"]), (
f"Grid Fast-Path selected wrong node type: {result.get('semantic')}"
)
assert any(
k in result.get("semantic_string", "").lower() for k in ["image button", "grid card layout container"]
), f"Grid Fast-Path selected wrong node type: {result.get('semantic_string')}"
def test_grid_fastpath_prefers_topmost_row(self):
"""
@@ -142,13 +169,15 @@ class TestExploreGridFastPath:
topmost one (smallest Y = row 1) since the intent says 'first'.
"""
engine = TelepathicEngine()
from unittest.mock import patch
with patch.object(engine, '_extract_semantic_nodes', return_value=EXPLORE_GRID_NODES):
with patch.object(engine, '_is_instagram_context', return_value=True):
result = engine.find_best_node("<fake>", "first image in explore grid", min_confidence=0.82, device=None)
with patch.object(engine, "_extract_semantic_nodes", return_value=EXPLORE_GRID_NODES):
with patch.object(engine, "_is_instagram_context", return_value=True):
result = engine.find_best_node(
"<fake>", "first image in explore grid", min_confidence=0.82, device=None
)
if result is not None:
# Row 1 items have y ≈ 559, Row 3 items have y ≈ 1523
assert result["y"] < 800, (
@@ -172,25 +201,26 @@ class TestVerifySuccessExploreGrid:
(no feed markers), verify_success must return None (Inconclusive).
"""
engine = TelepathicEngine()
# Simulate that we just clicked a grid item
TelepathicEngine._last_click_context = {
"intent": "first image in explore grid",
"semantic_string": "description: '4 photos by Patsy Weingart at row 1, column 1'",
"x": 178, "y": 559,
"timestamp": 1000
"x": 178,
"y": 559,
"timestamp": 1000,
}
# Post-click XML still shows the explore grid (no feed markers)
still_on_grid_xml = """
<node class="android.widget.FrameLayout">
<node resource-id="com.instagram.android:id/explore_action_bar" />
<node resource-id="com.instagram.android:id/grid_card_layout_container"
<node resource-id="com.instagram.android:id/grid_card_layout_container"
content-desc="4 photos by Patsy Weingart at row 1, column 1" />
<node resource-id="com.instagram.android:id/image_button" />
</node>
"""
result = engine.verify_success("first image in explore grid", still_on_grid_xml)
assert result is None, "verify_success should be inconclusive (None) when still on explore grid"
@@ -200,14 +230,15 @@ class TestVerifySuccessExploreGrid:
verify_success must return True.
"""
engine = TelepathicEngine()
TelepathicEngine._last_click_context = {
"intent": "first image in explore grid",
"semantic_string": "description: '4 photos by Patsy Weingart at row 1, column 1'",
"x": 178, "y": 559,
"timestamp": 1000
"x": 178,
"y": 559,
"timestamp": 1000,
}
# Post-click XML shows a feed post (has feed markers)
post_view_xml = """
<node class="android.widget.FrameLayout">
@@ -217,6 +248,6 @@ class TestVerifySuccessExploreGrid:
<node resource-id="com.instagram.android:id/row_feed_button_share" />
</node>
"""
result = engine.verify_success("first image in explore grid", post_view_xml)
assert result is True, "verify_success should pass when post view is visible"

62
tests/unit/test_goap_unlearn.py Normal file
View File

@@ -0,0 +1,62 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.goap import GoalExecutor
from GramAddict.core.screen_topology import ScreenType
def test_goap_unlearns_transition_on_back_press_trap():
"""Prove that GOAP forgets a specific topological transition when it gets trapped and spams 'back'."""
device = MagicMock()
orchestrator = GoalExecutor(device, "testuser")
# Mocking internal state
start_screen = ScreenType.HOME_FEED
goal = "open messages"
steps_taken = [
{"action": "tap explore tab"},
{"action": "press back"},
{"action": "press back"},
{"action": "press back"},
]
def mock_exec(*args, **kwargs):
print("EXECUTING:", args, kwargs)
return True
orchestrator._execute_action = MagicMock(side_effect=mock_exec) # "press back" succeeds
orchestrator.path_memory = MagicMock()
orchestrator.path_memory.recall_path.return_value = None
# To test the back-press circuit breaker, we just need to feed it 3 "press back" actions.
# Since achieve is complex, let's just test that the required logic
# exists inside it. The circuit breaker is in the "EXECUTE" block of achieve.
# We will mock the planner to return "press back" 3 times.
orchestrator.planner.plan_next_step = MagicMock(side_effect=[["press back"], ["press back"], ["press back"], []])
orchestrator.perceive = MagicMock(
return_value={"screen_type": ScreenType.EXPLORE_GRID, "available_actions": ["mock"]}
)
with patch("GramAddict.core.qdrant_memory.NavigationMemoryDB") as MockNavDB:
mock_nav_instance = MockNavDB.return_value
# We need to simulate that `steps_taken` already had "tap explore tab"
# However, achieve starts with an empty `steps_taken`.
# So we mock the internal variables if possible, but they are local.
# Alternatively, we make the planner return "tap explore tab", then 3 "press back"s.
orchestrator.planner.plan_next_step = MagicMock(
side_effect=["tap explore tab", "press back", "press back", "press back", "press back", None]
)
# Act
result = orchestrator.achieve(goal, max_steps=10)
# Assert (RED)
assert result is False
# Did it forget the path? (learn_path with success=False)
assert orchestrator.path_memory.learn_path.called
# Did it unlearn the transition?
print("MockNavDB calls:", MockNavDB.mock_calls)
print("NavInstance calls:", mock_nav_instance.mock_calls)
mock_nav_instance.unlearn_transition.assert_called_once_with(ScreenType.EXPLORE_GRID.value, "tap explore tab")

96
tests/unit/test_profile_interaction_edges.py
View File

@@ -1,12 +1,12 @@
import pytest
from unittest.mock import MagicMock
from GramAddict.core.telepathic_engine import TelepathicEngine
from GramAddict.core.q_nav_graph import QNavGraph
from GramAddict.core.telepathic_engine import TelepathicEngine
class TestProfileInteractionSync:
"""
TDD Tests: Reproduces the 'Already Followed -> Favorites' and 'No Story Ring'
TDD Tests: Reproduces the 'Already Followed -> Favorites' and 'No Story Ring'
bugs from 2026-04-17 live run.
"""
@@ -24,68 +24,84 @@ class TestProfileInteractionSync:
the engine must skip the click to prevent opening the Favorites/Mute bottom sheet.
"""
# Simulate a profile where the user is already followed
viable_nodes = [{
"semantic_string": "id context: 'profile header user action follow button', text: 'Following'",
"x": 500, "y": 600,
"width": 100, "height": 50,
"area": 5000,
"class_name": "android.widget.Button",
"resource_id": "com.instagram.android:id/button",
"original_attribs": {"text": "Following"}
}]
viable_nodes = [
{
"semantic_string": "id context: 'profile header user action follow button', text: 'Following'",
"x": 500,
"y": 600,
"width": 100,
"height": 50,
"area": 5000,
"class_name": "android.widget.Button",
"resource_id": "com.instagram.android:id/button",
"original_attribs": {"text": "Following"},
}
]
# Test vector-based matching fallback
self.engine._blacklist = {}
# Mock the extraction to avoid needing valid complex XML
self.engine._extract_semantic_nodes = MagicMock(return_value=viable_nodes)
# Mock the parsing instead of old extraction
self.engine._parser = MagicMock()
self.engine._parser.parse.return_value = MagicMock()
# We must return SpatialNode objects for the new architecture
from GramAddict.core.perception.spatial_parser import SpatialNode
mock_node = SpatialNode(MagicMock())
mock_node.resource_id = "com.instagram.android:id/button"
mock_node.text = "Following"
mock_node.bounds = [500, 600, 600, 650]
self.engine._parser.get_clickable_nodes.return_value = [mock_node]
self.engine._structural_sanity_check = MagicMock(return_value=True)
self.engine._is_instagram_context = MagicMock(return_value=True)
# Mock resolver to return our mock node
self.engine._resolver = MagicMock()
self.engine._resolver.resolve.return_value = mock_node
result = self.engine.find_best_node("<mock></mock>", "tap follow button on profile", device=self.mock_device)
# We must intercept it in TelepathicEngine before VLM is called
# Wait, find_best_node falls back to VLM if vector score is low.
# But if we inject it into memory, it triggers stage 1
self.engine._memory = {
"tap follow button on profile": ["id context: 'profile header user action follow button', text: 'Following'"]
}
TelepathicEngine._instance = self.engine
result = self.engine.find_best_node("<mock></mock>", "tap follow button on profile", device=self.mock_device)
assert result is not None, "Engine should return a skip result, not None"
assert result.get("skip") is True, "Must return skip: True to prevent Favorites menu from opening"
assert result.get("semantic") == "already_followed"
def test_story_ring_not_present_skips_click(self):
"""
If no story ring is explicitly in the XML, bot_flow should not execute
If no story ring is explicitly in the XML, bot_flow should not execute
the transition (simulated here by checking our XML evaluation logic).
"""
xml_without_story = '''
xml_without_story = """
<hierarchy>
<node resource-id="com.instagram.android:id/row_profile_header_imageview" content-desc="Profile picture" />
<node text="marisaundmarc" />
</hierarchy>
'''
has_story = "reel_ring" in xml_without_story or "unseen story" in xml_without_story.lower() or "story von" in xml_without_story.lower()
"""
has_story = (
"reel_ring" in xml_without_story
or "unseen story" in xml_without_story.lower()
or "story von" in xml_without_story.lower()
)
assert has_story is False, "Logic falsely identified a story when there is only a generic profile picture"
def test_story_ring_present_allows_click(self):
"""
If a story ring is present, the logic should allow the interaction.
"""
xml_with_story = '''
xml_with_story = """
<hierarchy>
<node resource-id="com.instagram.android:id/reel_ring" />
<node resource-id="com.instagram.android:id/row_profile_header_imageview" content-desc="mercedesbenz_de's unseen story" />
</hierarchy>
'''
has_story = "reel_ring" in xml_with_story or "unseen story" in xml_with_story.lower() or "story von" in xml_with_story.lower()
"""
has_story = (
"reel_ring" in xml_with_story
or "unseen story" in xml_with_story.lower()
or "story von" in xml_with_story.lower()
)
assert has_story is True, "Logic failed to identify active story ring"

79
tests/unit/test_sae_tesla_upgrade.py Normal file
View File

@@ -0,0 +1,79 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.situational_awareness import EscapeAction, SituationalAwarenessEngine, SituationEpisodeDB
class TestSAETeslaUpgrade:
def setup_method(self):
self.device_mock = MagicMock()
self.sae = SituationalAwarenessEngine(self.device_mock)
def test_sae_foreground_extraction(self):
"""
Ensures that modals/popups located at the END of the XML document
(highest Z-index) are prioritized in the compressed signature.
The old system truncated at elements[:50], missing the actual popup.
"""
# Create an XML with 60 background nodes, and 1 dialog node at the end
xml_dump = "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n<hierarchy rotation='0'>\n"
for i in range(60):
xml_dump += f' <node package="com.instagram.android" resource-id="id/feed_item_{i}" text="Feed {i}" bounds="[0,0][100,100]" />\n'
# The critical modal at the end
xml_dump += ' <node package="com.instagram.android" resource-id="id/dialog_container" text="Not Now" bounds="[100,100][200,200]" />\n'
xml_dump += "</hierarchy>"
compressed = self.sae._compress_xml(xml_dump)
# The compressed string MUST contain the dialog container
assert "dialog_container" in compressed, "Foreground extraction failed: modal was truncated!"
assert "Not Now" in compressed, "Foreground extraction failed: modal text was truncated!"
# It should prioritize the END of the document, so feed_item_0 should ideally be gone if capped at 50
assert "feed_item_0" not in compressed, "Background elements are still being prioritized over foreground!"
def test_sae_structural_generalization(self):
"""
Ensures that dynamic user content is stripped to allow cross-post modal generalization,
while short, static UI text (like "OK", "Cancel") is preserved.
"""
xml_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation='0'>
<node package="com.instagram.android" resource-id="id/comment" text="This is a very long user comment that changes every time we see this modal so it should be stripped!" />
<node package="com.instagram.android" resource-id="id/button" text="OK" />
</hierarchy>
"""
compressed = self.sae._compress_xml(xml_dump)
# Long dynamic text should be stripped or truncated to not pollute the vector space
assert "This is a very long user comment" not in compressed, "Dynamic text > 20 chars was not stripped!"
assert "text='<STRIPPED_DYNAMIC>'" in compressed or "This is a very lo" not in compressed
# Short static text should be kept
assert "OK" in compressed, "Short static UI text was incorrectly stripped!"
@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new=True)
def test_sae_negative_reinforcement(self):
"""
Ensures that failed escapes decay the confidence of the vector,
and eventually purge it, instead of just storing a useless 0.0 vector alongside it.
"""
db = SituationEpisodeDB()
# We need to mock db._db.client directly since it's an instance property
mock_client = MagicMock()
db._db._client = mock_client
db._db.client = mock_client
with patch.object(db._db, "upsert_point") as mock_upsert:
# Mock retrieve to return an existing point with confidence 0.4
mock_payload = {"confidence": 0.4, "action": {"action_type": "back", "x": 0, "y": 0, "reason": ""}}
mock_client.retrieve.return_value = [MagicMock(payload=mock_payload)]
# Simulate a FAILURE
action = EscapeAction("back")
db.learn("some_signature", action, success=False)
# Verify that it fetched the current confidence and updated it, or deleted it if < 0.1
# If confidence was 0.4 and delta is -0.5, it drops to -0.1 -> DELETED
mock_client.delete.assert_called_once()

22
tests/unit/test_sae_unlearn.py Normal file
View File

@@ -0,0 +1,22 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
def test_sae_has_unlearn_current_state():
"""Prove that SituationalAwarenessEngine exposes unlearn_current_state to heal from poisoned context."""
device = MagicMock()
sae = SituationalAwarenessEngine(device)
# Mocking internal compression and the screen_memory dependency
sae._compress_xml = MagicMock(return_value="<compressed_feed/>")
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB") as MockScreenMemoryDB:
mock_db_instance = MockScreenMemoryDB.return_value
# If unlearn_current_state does not exist, AttributeError (RED)
sae.unlearn_current_state("<full_feed_xml/>")
# Verify it compresses and delegates to purge_screen
sae._compress_xml.assert_called_once_with("<full_feed_xml/>")
mock_db_instance.purge_screen.assert_called_once_with("<compressed_feed/>")

50
tests/unit/test_self_healing_memory.py Normal file
View File

@@ -0,0 +1,50 @@
from unittest.mock import MagicMock, PropertyMock, patch
from GramAddict.core.qdrant_memory import NavigationMemoryDB, QdrantBase, ScreenMemoryDB
class DummyQdrantBase(QdrantBase):
def __init__(self):
self.client = MagicMock()
self.collection_name = "test_collection"
@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock)
def test_qdrant_base_has_delete_point(mock_is_connected):
"""Prove that QdrantBase implements delete_point for autonomous unlearning."""
mock_is_connected.return_value = True
db = DummyQdrantBase()
# If delete_point does not exist, this will raise AttributeError (RED)
db.generate_uuid = MagicMock(return_value="test-uuid-1234")
result = db.delete_point("test-seed")
# Assert
db.client.delete.assert_called_once_with(collection_name="test_collection", points_selector=["test-uuid-1234"])
assert result is True
@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock)
def test_screen_memory_has_purge_screen(mock_is_connected):
"""Prove that ScreenMemoryDB exposes purge_screen to heal poisoned classifications."""
mock_is_connected.return_value = True
db = ScreenMemoryDB()
db.client = MagicMock()
db.delete_point = MagicMock()
# If purge_screen does not exist, AttributeError (RED)
db.purge_screen("<node class='feed' />")
db.delete_point.assert_called_once_with("<node class='feed' />")
@patch("GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=PropertyMock)
def test_navigation_memory_has_unlearn_transition(mock_is_connected):
"""Prove that NavigationMemoryDB exposes unlearn_transition to destroy trap paths."""
mock_is_connected.return_value = True
db = NavigationMemoryDB()
db.client = MagicMock()
db.delete_point = MagicMock()
# If unlearn_transition does not exist, AttributeError (RED)
db.unlearn_transition("HOME_FEED", "tap explore tab")
db.delete_point.assert_called_once_with("HOME_FEED_tap explore tab")

38
tests/unit/test_telepathic_brevity_bonus.py
View File

@@ -1,38 +0,0 @@
from GramAddict.core.telepathic_engine import TelepathicEngine
def test_brevity_bonus_prioritizes_short_labels():
"""
Tests that the brevity bonus correctly prioritizes short, exact matches
over longer matches that contain the same keywords.
"""
engine = TelepathicEngine()
# A short, precise button
short_node = {
"x": 100,
"y": 200,
"area": 500,
"semantic_string": "text: 'Profile', id context: 'tab bar profile'",
"resource_id": "tab_bar_profile",
"original_attribs": {"desc": "", "text": "Profile"},
}
# A long, descriptive text that happens to contain "Profile"
long_node = {
"x": 100,
"y": 300,
"area": 5000,
"semantic_string": "text: 'Visit my profile to see more photos', id context: 'feed post text'",
"resource_id": "feed_post_text",
"original_attribs": {"desc": "", "text": "Visit my profile to see more photos"},
}
nodes = [long_node, short_node]
# "profile" is the intent
result = engine._keyword_match_score("profile", nodes)
assert result is not None, "Failed to extract node via fast path"
# The short node should win because of the brevity bonus (0.2)
assert "tab bar profile" in result["semantic"], "Brevity bonus failed to prioritize the shorter label"

65
tests/unit/test_telepathic_confidence.py
View File

@@ -1,65 +0,0 @@
import pytest
from GramAddict.core.telepathic_engine import TelepathicEngine
def test_extract_post_author_confidence():
"""
Tests that the TelepathicEngine can confidently extract the post author
header node from a standard feed XML dump, even if it falls back to the
fast path or embeddings.
"""
engine = TelepathicEngine()
# A generic Feed post author node
author_node = {
"x": 100, "y": 200, "area": 500,
"semantic_string": "description: 'fiona.dawson', id context: 'row feed photo profile name'",
"resource_id": "row_feed_photo_profile_name",
"original_attribs": {"desc": "fiona.dawson", "text": "fiona.dawson"}
}
# A generic Feed post image node
image_node = {
"x": 100, "y": 300, "area": 5000,
"semantic_string": "description: 'Post image', id context: 'row feed photo imageview'",
"resource_id": "row_feed_photo_imageview",
"original_attribs": {"desc": "Post image", "text": ""}
}
nodes = [author_node, image_node]
# The exact string used by _extract_post_content
result = engine._keyword_match_score("post author username header", nodes)
assert result is not None, "Failed to extract author node via fast path"
assert "fiona.dawson" in result["semantic"], "Extracted wrong node for author"
assert result["score"] >= 0.35, f"Confidence score too low: {result['score']}"
def test_extract_post_description_confidence():
"""
Tests that the TelepathicEngine can confidently extract the post description
node from a standard feed XML dump.
"""
engine = TelepathicEngine()
author_node = {
"x": 100, "y": 200, "area": 500,
"semantic_string": "description: 'fiona.dawson', id context: 'row feed photo profile name'",
"resource_id": "row_feed_photo_profile_name",
"original_attribs": {"desc": "fiona.dawson", "text": "fiona.dawson"}
}
image_node = {
"x": 100, "y": 300, "area": 5000,
"semantic_string": "description: 'Post image', id context: 'row feed photo imageview'",
"resource_id": "row_feed_photo_imageview",
"original_attribs": {"desc": "Post image", "text": ""}
}
nodes = [author_node, image_node]
# The exact string used by _extract_post_content
result = engine._keyword_match_score("post image video media content description", nodes)
assert result is not None, "Failed to extract image/media node via fast path"
assert "imageview" in result["semantic"], "Extracted wrong node for media"
assert result["score"] >= 0.35, f"Confidence score too low: {result['score']}"