Compare commits
2 Commits
4bc9183901
...
5fbbe3d273
| Author | SHA1 | Date | |
|---|---|---|---|
| 5fbbe3d273 | |||
| f85d0a8a76 |
@@ -92,9 +92,8 @@ def check_production_integrity():
|
||||
"""
|
||||
import sys
|
||||
|
||||
# If we are in a pytest session, we expect and allow mocks
|
||||
if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
|
||||
return
|
||||
# We no longer skip this in tests. Production integrity must hold everywhere.
|
||||
pass
|
||||
|
||||
try:
|
||||
from unittest.mock import MagicMock
|
||||
@@ -925,6 +924,12 @@ def _run_zero_latency_feed_loop(
|
||||
|
||||
elif governance_decision == "CHECK_CURIOSITY":
|
||||
logger.info("👀 [Curiosity] Spontaneously checking DMs / Notifications...")
|
||||
|
||||
# 🛡️ Structural Guard: Curiosity targets (DMs, Notifications) are ONLY available on HomeFeed.
|
||||
# We must navigate there first, breaking current context.
|
||||
nav_graph.navigate_to("HomeFeed", zero_engine)
|
||||
sleep(random.uniform(1.0, 2.5))
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
explore_target = random.choice(["MessageInbox", "Notifications"])
|
||||
|
||||
@@ -17,13 +17,7 @@ class Config:
|
||||
self.args = kwargs
|
||||
self.module = True
|
||||
else:
|
||||
# Avoid parsing sys.argv if we are running in a test environment (pytest)
|
||||
# as pytest arguments will cause argparse to fail with SystemExit: 2
|
||||
is_pytest = "pytest" in sys.modules
|
||||
if is_pytest:
|
||||
self.args = []
|
||||
else:
|
||||
self.args = list(sys.argv)
|
||||
self.args = list(sys.argv)
|
||||
self.module = False
|
||||
|
||||
if not self.module and "--config" not in self.args:
|
||||
|
||||
@@ -86,6 +86,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
identity_engine = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
identity_engine.device = device
|
||||
screen_info = identity_engine.identify(xml_dump)
|
||||
|
||||
screen_type = screen_info["screen_type"]
|
||||
@@ -220,6 +221,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
@@ -246,6 +248,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
|
||||
@@ -63,6 +63,7 @@ class GoalExecutor:
|
||||
self.device = device
|
||||
self.username = bot_username
|
||||
self.screen_id = ScreenIdentity(bot_username)
|
||||
self.screen_id.device = device
|
||||
self.planner = GoalPlanner(bot_username)
|
||||
self.path_memory = PathMemory(bot_username)
|
||||
self.max_steps = 15 # Safety: never execute more than 15 steps
|
||||
@@ -239,7 +240,8 @@ class GoalExecutor:
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
keys_to_clear = [
|
||||
k for k in self.action_failures.keys()
|
||||
k
|
||||
for k in self.action_failures.keys()
|
||||
if k[0] == screen_type and ScreenTopology.is_structural_action(screen_type, k[1])
|
||||
]
|
||||
for k in keys_to_clear:
|
||||
|
||||
@@ -69,10 +69,11 @@ class IntentResolver:
|
||||
# - "tap profile tab", "tap home tab", "explore tab"
|
||||
# - NOT "exclude bottom tabs", "tabbar", random mentions
|
||||
import re
|
||||
|
||||
_TAB_PATTERN = re.compile(
|
||||
r"\btap\s+\w+\s+tab\b" # "tap profile tab", "tap home tab"
|
||||
r"|\b\w+\s+tab\b" # "profile tab", "explore tab"
|
||||
r"|^tab\b", # "tab" at start of intent
|
||||
r"\btap\s+\w+\s+tab\b" # "tap profile tab", "tap home tab"
|
||||
r"|\b\w+\s+tab\b" # "profile tab", "explore tab"
|
||||
r"|^tab\b", # "tab" at start of intent
|
||||
re.IGNORECASE,
|
||||
)
|
||||
filtered = []
|
||||
@@ -277,6 +278,27 @@ class IntentResolver:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
|
||||
return node
|
||||
|
||||
# --- Navigation Tab Fast-Paths ---
|
||||
# Deterministically identify bottom navigation tabs to prevent VLM confusion
|
||||
tab_map = {
|
||||
"home tab": "feed_tab",
|
||||
"feed tab": "feed_tab",
|
||||
"reels tab": "clips_tab",
|
||||
"clips tab": "clips_tab",
|
||||
"explore tab": "search_tab",
|
||||
"search tab": "search_tab",
|
||||
"profile tab": "profile_tab",
|
||||
"message tab": "direct_tab",
|
||||
"direct tab": "direct_tab",
|
||||
}
|
||||
for intent_key, resource_suffix in tab_map.items():
|
||||
if intent_key in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if rid.endswith(f":id/{resource_suffix}"):
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
|
||||
return node
|
||||
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
# we strictly filter candidates to those whose text or content_desc contains the quote.
|
||||
|
||||
@@ -43,7 +43,7 @@ class ScreenIdentity:
|
||||
except ImportError:
|
||||
self.screen_memory = None
|
||||
|
||||
def identify(self, xml_dump: str) -> Dict[str, Any]:
|
||||
def identify(self, xml_dump: str, screenshot_b64: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Analyzes an XML dump and returns a complete screen description.
|
||||
|
||||
@@ -116,6 +116,11 @@ class ScreenIdentity:
|
||||
}
|
||||
)
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
sae = SituationalAwarenessEngine.get_instance()
|
||||
signature = sae._compress_xml(xml_dump) if sae else self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Foreign app check ──
|
||||
if app_id not in packages:
|
||||
return {
|
||||
@@ -123,18 +128,16 @@ class ScreenIdentity:
|
||||
"available_actions": ["press back", "force start instagram"],
|
||||
"selected_tab": None,
|
||||
"context": {"packages": list(packages)},
|
||||
"signature": self._compute_signature(resource_ids, content_descs, texts),
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
desc_lower = " ".join(content_descs).lower()
|
||||
text_lower = " ".join(texts).lower()
|
||||
ids_str = " ".join(resource_ids).lower()
|
||||
|
||||
signature = self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Identify screen type from structural signals ──
|
||||
screen_type = self._classify_screen(
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature, screenshot_b64
|
||||
)
|
||||
|
||||
# ── Extract available actions from clickable elements ──
|
||||
@@ -153,16 +156,36 @@ class ScreenIdentity:
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None):
|
||||
"""Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
def _classify_screen(
|
||||
self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None, screenshot_b64=None
|
||||
):
|
||||
"""
|
||||
Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
|
||||
# Priority 0: Content-creation overlays that block ALL navigation.
|
||||
# Priority 0: Check Qdrant Semantic Cache (Learned Truth/LLM Overrides)
|
||||
# This MUST be checked first. If the LLM declared this specific layout a "false positive"
|
||||
# and cached it as NORMAL, it must override any rigid structural heuristics below to prevent
|
||||
# infinite loops.
|
||||
is_normal_override = False
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
if cached_type_str == "NORMAL":
|
||||
is_normal_override = True
|
||||
else:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Priority 1: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
if not is_normal_override:
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 1: Structural Heuristics (100% Deterministic)
|
||||
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
|
||||
@@ -225,50 +248,52 @@ class ScreenIdentity:
|
||||
if "message_input" in ids:
|
||||
return ScreenType.DM_INBOX # Fallback for DM thread as inbox
|
||||
|
||||
# Priority 2: Check Qdrant Semantic Cache (Fuzzy/VLM derived)
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
# End of structural heuristics
|
||||
|
||||
# Priority 3: Semantic VLM Classification Fallback
|
||||
if not screenshot_b64 and getattr(self, "device", None) is not None:
|
||||
screenshot_b64 = self.device.get_screenshot_b64()
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "llava:latest") if hasattr(cfg, "args") else "llava:latest"
|
||||
|
||||
layout_context = (
|
||||
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
|
||||
)
|
||||
prompt = (
|
||||
f"Identify the Instagram screen layout type based on these DOM structural signals.\n"
|
||||
f"Identify the Instagram screen layout type based on the provided screenshot and structural signals.\n"
|
||||
f"Valid types: {[t.name for t in ScreenType]}\n"
|
||||
f"Context:\n{layout_context}\n"
|
||||
f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches."
|
||||
)
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False
|
||||
response = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt=prompt,
|
||||
user_prompt="Classify this screen layout.",
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
use_local_edge=True,
|
||||
)
|
||||
if response and isinstance(response, str):
|
||||
result = response.strip().upper()
|
||||
elif response and isinstance(response, dict) and "response" in response:
|
||||
result = response["response"].strip().upper()
|
||||
else:
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
result = response.strip().upper() if response else "UNKNOWN"
|
||||
|
||||
for t in ScreenType:
|
||||
if t.name in result:
|
||||
if is_normal_override and t == ScreenType.MODAL:
|
||||
# Prevent the LLM from hallucinating an obstacle if explicitly verified as NORMAL
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
if signature and self.screen_memory:
|
||||
self.screen_memory.store_screen(signature, t.name)
|
||||
return t
|
||||
|
||||
@@ -13,7 +13,8 @@ class PersistentList(list):
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
path = f"accounts/{self.filename}.json"
|
||||
base_dir = os.environ.get("GRAMADDICT_ACCOUNTS_DIR", "accounts")
|
||||
path = f"{base_dir}/{self.filename}.json"
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
@@ -27,9 +28,8 @@ class PersistentList(list):
|
||||
self.persist()
|
||||
|
||||
def persist(self, directory=None):
|
||||
if os.environ.get("PYTEST_CURRENT_TEST"):
|
||||
return
|
||||
folder = f"accounts/{directory}" if directory else "accounts"
|
||||
base_dir = os.environ.get("GRAMADDICT_ACCOUNTS_DIR", "accounts")
|
||||
folder = f"{base_dir}/{directory}" if directory else base_dir
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
path = f"{folder}/{self.filename}.json"
|
||||
try:
|
||||
|
||||
@@ -184,6 +184,7 @@ class QdrantBase:
|
||||
self.client.upsert(
|
||||
collection_name=self.collection_name,
|
||||
points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)],
|
||||
wait=True,
|
||||
)
|
||||
|
||||
# ABSOLUTE LOGGING: User requirement for full observability
|
||||
@@ -431,7 +432,7 @@ class UIMemoryDB(QdrantBase):
|
||||
if eval_result:
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
# If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
|
||||
@@ -462,7 +463,7 @@ class UIMemoryDB(QdrantBase):
|
||||
if eval_result:
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
return None
|
||||
@@ -515,7 +516,7 @@ class UIMemoryDB(QdrantBase):
|
||||
)
|
||||
logger.info(
|
||||
f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[35m"} # Magenta color
|
||||
extra={"color": "\x1b[35m"}, # Magenta color
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Qdrant storage error: {e}")
|
||||
@@ -582,7 +583,7 @@ class UIMemoryDB(QdrantBase):
|
||||
symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
|
||||
logger.info(
|
||||
f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color}
|
||||
extra={"color": color},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Confidence adjustment error: {e}")
|
||||
|
||||
@@ -270,7 +270,13 @@ class SituationalAwarenessEngine:
|
||||
if clickable == "true":
|
||||
parts.append("CLICKABLE")
|
||||
if bounds:
|
||||
parts.append(f"bounds={bounds}")
|
||||
nums = [int(n) for n in re.findall(r"\d+", bounds)]
|
||||
if len(nums) == 4:
|
||||
cx = (nums[0] + nums[2]) // 2
|
||||
cy = (nums[1] + nums[3]) // 2
|
||||
parts.append(f"bounds={bounds} center=({cx},{cy})")
|
||||
else:
|
||||
parts.append(f"bounds={bounds}")
|
||||
|
||||
elements.append(" | ".join(parts))
|
||||
|
||||
@@ -406,11 +412,21 @@ class SituationalAwarenessEngine:
|
||||
|
||||
compressed = self._compress_xml(xml_dump)
|
||||
|
||||
cached_type = screen_memory.get_screen_type(compressed)
|
||||
|
||||
if cached_type:
|
||||
if cached_type == "OBSTACLE_MODAL":
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
elif cached_type == "NORMAL":
|
||||
return SituationType.NORMAL
|
||||
|
||||
# ── Structural Fast-Check: Content-Creation Overlays ──
|
||||
# These full-screen overlays live INSIDE Instagram's package but block
|
||||
# all normal navigation. They are invisible to the foreign-app detector
|
||||
# and frequently fool the LLM into thinking they are "normal" browsing.
|
||||
# Detecting them structurally is O(1) and requires ZERO LLM calls.
|
||||
# This is checked AFTER Qdrant to ensure that if the LLM unlearned a false positive,
|
||||
# we respect the learned NORMAL state and don't infinite-loop.
|
||||
creation_flow_markers = (
|
||||
"quick_capture", # Camera / story capture overlay
|
||||
"gallery_cancel_button", # Story gallery "Back to Home" button
|
||||
@@ -427,14 +443,6 @@ class SituationalAwarenessEngine:
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
|
||||
cached_type = screen_memory.get_screen_type(compressed)
|
||||
|
||||
if cached_type:
|
||||
if cached_type == "OBSTACLE_MODAL":
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
elif cached_type == "NORMAL":
|
||||
return SituationType.NORMAL
|
||||
|
||||
# If not cached, query LLM for autonomous structural classification
|
||||
try:
|
||||
from GramAddict.core.config import Config
|
||||
@@ -442,7 +450,7 @@ class SituationalAwarenessEngine:
|
||||
|
||||
prompt = (
|
||||
"You are a Situation Classifier for a mobile automation agent.\n"
|
||||
"Analyze the given Android UI XML dump. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"Analyze the given Android UI XML dump AND screenshot. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"covering the screen that needs to be dismissed, or is this a NORMAL usable screen?\n"
|
||||
"A 'clean_sheet_container' with standard Instagram feed content is NORMAL.\n"
|
||||
"A survey, rating prompt, 'not now' prompt, or permission dialog is an OBSTACLE_MODAL.\n"
|
||||
@@ -459,11 +467,17 @@ class SituationalAwarenessEngine:
|
||||
args = Config().args
|
||||
except Exception:
|
||||
pass
|
||||
model = getattr(args, "ai_model", "qwen3.5:latest")
|
||||
url = getattr(args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
res = query_telepathic_llm(
|
||||
model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict JSON classifier.",
|
||||
user_prompt=prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
use_local_edge=True,
|
||||
)
|
||||
import json
|
||||
|
||||
@@ -504,27 +518,31 @@ class SituationalAwarenessEngine:
|
||||
Called ONLY when recall AND structural planning both miss.
|
||||
"""
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
try:
|
||||
args = Config().args
|
||||
model = getattr(args, "ai_fallback_model", "llama3.2:1b")
|
||||
url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
except Exception:
|
||||
model = "llama3.2:1b"
|
||||
model = "llava:latest"
|
||||
url = "http://localhost:11434/api/generate"
|
||||
|
||||
system_prompt = (
|
||||
"You are an Android UI navigation agent. Your job is to escape obstacles "
|
||||
"(dialogs, modals, foreign apps, system popups) and return to Instagram. "
|
||||
"Analyze the screen content and return a JSON escape action.\n\n"
|
||||
"Analyze the screen content (Screenshot AND XML) and return a JSON escape action.\n\n"
|
||||
"Rules:\n"
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is OBSTACLE_LOCKED_SCREEN, action must be 'unlock'\n"
|
||||
"- If the Situation type is OBSTACLE_FOREIGN_APP, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_locked_screen, action must be 'unlock'\n"
|
||||
"- If the Situation type is obstacle_foreign_app, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_system, you MUST look for 'Deny', 'Don't allow', or 'Cancel' and click it. \n"
|
||||
" NEVER click 'Allow', 'OK', or 'Confirm' on system permissions.\n"
|
||||
" If no negative action button exists, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- If nothing else works, suggest 'app_start' to force-reopen Instagram\n"
|
||||
"- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n"
|
||||
"- When you choose to click, you MUST use the EXACT coordinates provided in `center=(x,y)` for that element in the XML\n"
|
||||
'- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}'
|
||||
)
|
||||
|
||||
@@ -535,20 +553,31 @@ class SituationalAwarenessEngine:
|
||||
user_prompt += "What action should I take to clear this obstacle and return to Instagram? Return JSON only."
|
||||
|
||||
try:
|
||||
resp = query_llm(
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
|
||||
resp = query_telepathic_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=True,
|
||||
timeout=30,
|
||||
max_tokens=300,
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
)
|
||||
if resp and "response" in resp:
|
||||
if resp:
|
||||
import json
|
||||
|
||||
data = json.loads(resp["response"])
|
||||
try:
|
||||
data = json.loads(resp)
|
||||
except json.JSONDecodeError:
|
||||
# Try extracting JSON via regex if LLM was chatty
|
||||
import re
|
||||
|
||||
match = re.search(r"\{.*\}", resp, re.DOTALL)
|
||||
if match:
|
||||
data = json.loads(match.group(0))
|
||||
else:
|
||||
raise ValueError(f"Could not parse JSON from: {resp}")
|
||||
|
||||
return EscapeAction(
|
||||
action_type=data.get("action", "back"),
|
||||
x=int(data.get("x", 0)),
|
||||
|
||||
33
test_llm_false_positive.py
Normal file
33
test_llm_false_positive.py
Normal file
@@ -0,0 +1,33 @@
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
xml = """
|
||||
<node package="com.instagram.android">
|
||||
<node resource-id="com.instagram.android:id/gallery_cancel_button" bounds="[10,10][20,20]" />
|
||||
<node resource-id="com.instagram.android:id/feed_tab" selected="true" bounds="[0,0][1080,2400]" />
|
||||
</node>
|
||||
"""
|
||||
|
||||
system_prompt = (
|
||||
"You are an Android UI navigation agent. Your job is to escape obstacles "
|
||||
"(dialogs, modals, foreign apps, system popups) and return to Instagram. "
|
||||
"Analyze the screen content (Screenshot AND XML) and return a JSON escape action.\n\n"
|
||||
"Rules:\n"
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is OBSTACLE_LOCKED_SCREEN, action must be 'unlock'\n"
|
||||
"- If the Situation type is OBSTACLE_FOREIGN_APP, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- 'reason' must explain why.\n"
|
||||
'Output ONLY valid JSON matching: {"action": "click"|"back"|"unlock"|"false_positive", "x": int, "y": int, "reason": str}'
|
||||
)
|
||||
user_prompt = f"Situation: OBSTACLE_MODAL\n\nXML Hierarchy:\n{xml}\n\nWhat action should I take to clear this obstacle and return to Instagram? Return JSON only."
|
||||
|
||||
print(
|
||||
query_telepathic_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="llava:latest",
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
images_b64=None,
|
||||
temperature=0.0,
|
||||
)
|
||||
)
|
||||
@@ -16,7 +16,6 @@ import time
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core import utils
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -135,22 +134,18 @@ def iteration_guard():
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def isolated_screen_memory(monkeypatch):
|
||||
"""Ensures we use a separate Qdrant collection for E2E tests and clean it.
|
||||
This replaces the old Qdrant mock so tests use the REAL database."""
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def session_env(tmp_path_factory):
|
||||
"""
|
||||
Forces production parity by using real logic with local-only backends.
|
||||
"""
|
||||
# 1. Honest Qdrant: Use real QdrantClient with in-memory storage
|
||||
os.environ["QDRANT_URL"] = ":memory:"
|
||||
|
||||
def test_init(self, *args, **kwargs):
|
||||
super(ScreenMemoryDB, self).__init__(collection_name="test_e2e_screens")
|
||||
|
||||
monkeypatch.setattr(ScreenMemoryDB, "__init__", test_init)
|
||||
|
||||
db = ScreenMemoryDB()
|
||||
if db.is_connected:
|
||||
db.wipe_collection()
|
||||
|
||||
yield db
|
||||
# 2. Honest Persistence: Use a temporary directory for accounts
|
||||
accounts_dir = tmp_path_factory.mktemp("accounts")
|
||||
os.environ["GRAMADDICT_ACCOUNTS_DIR"] = str(accounts_dir)
|
||||
os.makedirs(os.path.join(str(accounts_dir), "testuser"), exist_ok=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
@@ -470,31 +465,7 @@ def _patch_module_delays(monkeypatch, module_path: str, sleep_fn, random_sleep_f
|
||||
monkeypatch.setattr(mod.random, "uniform", lambda a, b: float(a))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_all_delays(monkeypatch, request):
|
||||
"""Replaces all humanized hardware delays with no-ops."""
|
||||
if request.config.getoption("--live"):
|
||||
return
|
||||
|
||||
def money_sleep(*args, **kwargs):
|
||||
pass
|
||||
|
||||
def random_sleep(*args, **kwargs):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(time, "sleep", money_sleep)
|
||||
monkeypatch.setattr(utils, "random_sleep", random_sleep)
|
||||
monkeypatch.setattr(utils, "sleep", money_sleep)
|
||||
if hasattr(utils, "random"):
|
||||
monkeypatch.setattr(utils.random, "uniform", lambda a, b: float(a))
|
||||
|
||||
# Each module gets its own try-block so a missing attribute in one
|
||||
# doesn't prevent patching the others.
|
||||
_patch_module_delays(monkeypatch, "GramAddict.core.bot_flow", money_sleep, random_sleep)
|
||||
_patch_module_delays(monkeypatch, "GramAddict.core.q_nav_graph", money_sleep, random_sleep)
|
||||
_patch_module_delays(monkeypatch, "GramAddict.core.goap", money_sleep, random_sleep)
|
||||
_patch_module_delays(monkeypatch, "GramAddict.core.device_facade", money_sleep, random_sleep)
|
||||
_patch_module_delays(monkeypatch, "GramAddict.core.darwin_engine", money_sleep, random_sleep)
|
||||
# Note: mock_all_delays removed to favor production 'speed_multiplier' logic.
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -526,7 +497,7 @@ def e2e_configs():
|
||||
stories_percentage=100,
|
||||
working_hours=[0.0, 24.0],
|
||||
time_delta_session=0,
|
||||
speed_multiplier=1.0,
|
||||
speed_multiplier=100.0,
|
||||
disable_filters=False,
|
||||
interaction_users_amount="1",
|
||||
scrape_profiles=False,
|
||||
@@ -653,6 +624,7 @@ class E2EDeviceStub:
|
||||
self.pressed_keys = []
|
||||
self.clicks = []
|
||||
self.swipes = []
|
||||
self.app_starts = []
|
||||
self.app_id = "com.instagram.android"
|
||||
self._info = {
|
||||
"screenOn": True,
|
||||
@@ -748,7 +720,7 @@ class E2EDeviceStub:
|
||||
self.swipes.append({"start": (sx, sy), "end": (ex, ey)})
|
||||
|
||||
def app_start(self, pkg, use_monkey=False):
|
||||
pass
|
||||
self.app_starts.append(pkg)
|
||||
|
||||
def app_stop(self, pkg):
|
||||
pass
|
||||
|
||||
@@ -7,11 +7,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.ad_guard import AdGuardPlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_ad_guard_detects_sponsored_post(make_real_device_with_image):
|
||||
def test_ad_guard_detects_sponsored_post(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: AdGuardPlugin must successfully identify a sponsored post
|
||||
in a real feed using the TelepathicEngine.
|
||||
@@ -24,24 +23,20 @@ def test_ad_guard_detects_sponsored_post(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(jpg_path, xml)
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace()
|
||||
session_state = SessionState(configs)
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_ad_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = AdGuardPlugin()
|
||||
|
||||
@@ -8,11 +8,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.scrape_profile import ScrapeProfilePlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
def test_scrape_profile_extracts_data_correctly(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: ScrapeProfilePlugin must use the TelepathicEngine to correctly
|
||||
identify the Follower count, Following count, and Bio text nodes on a real profile.
|
||||
@@ -25,37 +24,33 @@ def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(jpg_path, xml)
|
||||
|
||||
# Create dummy config and session state
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
scrape_profiles=True,
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
# Override only what the test needs — fixture provides all production defaults
|
||||
e2e_configs.args.scrape_profiles = True
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
# Initialize Telepathic Engine
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
# Create behavior context
|
||||
class DummyCRM:
|
||||
def __init__(self):
|
||||
self.last_enriched_data = None
|
||||
# Spy wrapper: capture enrichment data for assertions while exercising real CRM
|
||||
crm = cognitive_stack["crm"]
|
||||
_original_enrich = crm.enrich_lead
|
||||
_enriched_data = {}
|
||||
|
||||
def enrich_lead(self, username, data):
|
||||
self.last_enriched_data = data
|
||||
def _spy_enrich_lead(username, data):
|
||||
_enriched_data.update(data)
|
||||
_original_enrich(username, data)
|
||||
|
||||
crm.enrich_lead = _spy_enrich_lead
|
||||
|
||||
crm = DummyCRM()
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_scrape_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic, "crm": crm},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = ScrapeProfilePlugin()
|
||||
@@ -64,10 +59,10 @@ def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, "ScrapeProfilePlugin did not execute successfully"
|
||||
assert crm.last_enriched_data is not None, "CRM enrich_lead was not called"
|
||||
assert len(_enriched_data) > 0, "CRM enrich_lead was not called"
|
||||
|
||||
# Check the scraped data accuracy
|
||||
data = crm.last_enriched_data
|
||||
data = _enriched_data
|
||||
|
||||
assert data["username"] == "test_scrape_user"
|
||||
|
||||
|
||||
@@ -7,12 +7,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.story_view import StoryViewPlugin
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_story_view_clicks_story_ring(make_real_device_with_image):
|
||||
def test_story_view_clicks_story_ring(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: StoryViewPlugin must correctly identify if a story exists
|
||||
and trigger the 'tap story ring avatar' navigation.
|
||||
@@ -46,30 +44,23 @@ def test_story_view_clicks_story_ring(make_real_device_with_image):
|
||||
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_before, xml_after, xml_after, xml_after]
|
||||
)
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
stories_percentage=100, # Force it to run
|
||||
stories_count="1",
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
# Override only what the test needs — fixture provides all production defaults
|
||||
e2e_configs.args.stories_percentage = 100
|
||||
e2e_configs.args.stories_count = "1"
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
# Use real NavGraph
|
||||
nav_graph = QNavGraph(device)
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_story_user",
|
||||
context_xml=xml_before,
|
||||
cognitive_stack={"telepathic": telepathic, "nav_graph": nav_graph},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = StoryViewPlugin()
|
||||
|
||||
101
tests/e2e/test_bot_flow_curiosity.py
Normal file
101
tests/e2e/test_bot_flow_curiosity.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
Tests for Bot Flow Curiosity Logic
|
||||
==================================
|
||||
Ensures that the spontaneous CHECK_CURIOSITY feature correctly shifts context
|
||||
and relies on structural safety (navigating to HomeFeed) rather than hallucinating
|
||||
clicks on invalid screens like Profiles.
|
||||
"""
|
||||
|
||||
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
|
||||
from GramAddict.core.session_state import SessionState
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
from tests.e2e.device_emulator import create_emulator_facade
|
||||
|
||||
HOME_FEED_XML = load_fixture_xml("home_feed_real.xml")
|
||||
PROFILE_XML = load_fixture_xml("other_profile_real.xml")
|
||||
|
||||
|
||||
def _build_curiosity_state_machine():
|
||||
states = {
|
||||
"home_feed": HOME_FEED_XML,
|
||||
"other_profile": PROFILE_XML,
|
||||
}
|
||||
|
||||
transitions = {
|
||||
"home_feed": {
|
||||
"clicks": [
|
||||
({"id": "com.instagram.android:id/row_feed_photo_profile_name"}, "other_profile"),
|
||||
],
|
||||
"press": [
|
||||
("back", "home_feed"),
|
||||
],
|
||||
},
|
||||
"other_profile": {
|
||||
"clicks": [
|
||||
({"desc": "Home"}, "home_feed"),
|
||||
({"id": "com.instagram.android:id/feed_tab"}, "home_feed"),
|
||||
],
|
||||
"press": [
|
||||
("back", "home_feed"),
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
return states, transitions
|
||||
|
||||
|
||||
def test_curiosity_navigates_to_homefeed_before_checking(
|
||||
e2e_configs, e2e_cognitive_stack_factory, setup_e2e_plugin_registry, monkeypatch
|
||||
):
|
||||
"""
|
||||
Simulates the feed loop starting on other_profile.
|
||||
When CHECK_CURIOSITY triggers, it must FIRST navigate to HomeFeed before
|
||||
trying to 'tap heart icon notifications'.
|
||||
"""
|
||||
# We do NOT mock production code. No bf.sleep or _humanized_scroll mocking.
|
||||
import random
|
||||
import time
|
||||
|
||||
random.seed(31) # First random.random() is 0.057 < 0.06, triggers CHECK_CURIOSITY
|
||||
|
||||
# 1. Setup emulator starting on other_profile
|
||||
states, transitions = _build_curiosity_state_machine()
|
||||
device, emulator = create_emulator_facade("other_profile", states, transitions, monkeypatch)
|
||||
|
||||
# 2. Setup REAL Config & SessionState
|
||||
configs = e2e_configs
|
||||
|
||||
session_state = SessionState(configs)
|
||||
|
||||
# 4. Setup Cognitive Stack
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
nav_graph = cognitive_stack["nav_graph"]
|
||||
zero_engine = cognitive_stack["zero_engine"]
|
||||
dopamine = cognitive_stack["dopamine"]
|
||||
|
||||
# We want it to exit cleanly after executing a few loops, without hanging forever.
|
||||
dopamine.session_limit_seconds = 0.1
|
||||
dopamine.session_start = time.time()
|
||||
|
||||
# 5. Run the loop (starting from other_profile)
|
||||
try:
|
||||
_run_zero_latency_feed_loop(
|
||||
device=device,
|
||||
zero_engine=zero_engine,
|
||||
nav_graph=nav_graph,
|
||||
configs=configs,
|
||||
session_state=session_state,
|
||||
job_target="homefeed",
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
except Exception:
|
||||
# It will likely crash when trying to 'tap heart icon notifications'
|
||||
# because the emulator state machine doesn't define what that button does.
|
||||
# This is expected and perfectly fine, because it proves the bot TRIED to
|
||||
# tap it AFTER navigating.
|
||||
pass
|
||||
|
||||
# 6. Assertions: Must navigate to HomeFeed FIRST
|
||||
assert emulator.current_state == "home_feed", (
|
||||
"Curiosity failed to navigate to HomeFeed before executing! " f"Bot is stuck on {emulator.current_state}"
|
||||
)
|
||||
@@ -7,10 +7,11 @@ purged so it doesn't immediately trap itself again on the next iteration.
|
||||
"""
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
from tests.e2e.device_emulator import create_emulator_facade
|
||||
|
||||
def test_goap_recovers_from_trapped_state_with_restart(e2e_device, monkeypatch):
|
||||
|
||||
def test_goap_recovers_from_trapped_state_with_restart(monkeypatch):
|
||||
"""
|
||||
Simulates a broken UI where an action repeatedly fails.
|
||||
Verifies that GOAP triggers 'force start instagram', and successfully
|
||||
@@ -18,58 +19,38 @@ def test_goap_recovers_from_trapped_state_with_restart(e2e_device, monkeypatch):
|
||||
avoiding an infinite restart loop.
|
||||
"""
|
||||
xml = load_fixture_xml("home_feed_real.xml")
|
||||
|
||||
# We create a device that ALWAYS returns the same home_feed XML,
|
||||
# no matter what action is taken. This forces the UI to never change,
|
||||
# which causes actions to fail repeatedly.
|
||||
device = e2e_device([xml] * 20)
|
||||
|
||||
|
||||
# Single-state emulator with no transitions → UI never changes.
|
||||
# This forces actions to fail repeatedly, triggering GOAP trap detection.
|
||||
device, emulator = create_emulator_facade("trapped", {"trapped": xml}, {}, monkeypatch)
|
||||
|
||||
goap = GoalExecutor.get_instance(device, bot_username="testuser")
|
||||
goap.action_failures.clear()
|
||||
|
||||
# We intercept app_start to track how many times it was forced to restart
|
||||
restart_count = 0
|
||||
def mock_app_start(*args, **kwargs):
|
||||
nonlocal restart_count
|
||||
restart_count += 1
|
||||
|
||||
monkeypatch.setattr(device, "app_start", mock_app_start)
|
||||
|
||||
|
||||
# We want to ask it to reach REELS_FEED.
|
||||
# Since the UI never changes, 'tap reels tab' will fail.
|
||||
# It will fail twice, get masked, GOAP gets trapped, triggers restart.
|
||||
# We set max_steps to a small number so it doesn't loop forever in the test.
|
||||
# We just want to see that AFTER the restart, it tries 'tap reels tab' again,
|
||||
# meaning it cleared the state.
|
||||
|
||||
# Let's mock _execute_action slightly just to spy on it without changing behavior
|
||||
original_execute = goap._execute_action
|
||||
executed_actions = []
|
||||
|
||||
def spy_execute(action, goal=None):
|
||||
executed_actions.append(action)
|
||||
return original_execute(action, goal)
|
||||
|
||||
monkeypatch.setattr(goap, "_execute_action", spy_execute)
|
||||
|
||||
|
||||
goap.achieve("open reels", max_steps=6)
|
||||
|
||||
|
||||
# It should have tried 'tap reels tab' twice, failed both times,
|
||||
# then triggered 'force start instagram'.
|
||||
# Because of the fix, after the restart, it should have cleared failures
|
||||
# and tried 'tap reels tab' AGAIN.
|
||||
|
||||
assert restart_count > 0, "GOAP never attempted to force restart Instagram when trapped!"
|
||||
|
||||
# Count how many times it tried the action
|
||||
reels_attempts = executed_actions.count("tap reels tab")
|
||||
|
||||
|
||||
assert len(emulator.app_starts) > 0, "GOAP never attempted to force restart Instagram when trapped!"
|
||||
|
||||
# Count how many times it clicked the screen (trying to tap the reels tab)
|
||||
reels_attempts = len(emulator.clicks)
|
||||
|
||||
assert reels_attempts > 2, (
|
||||
f"GOAP got trapped, restarted, but never tried the action again! "
|
||||
f"It only tried {reels_attempts} times, meaning the memory leak is still there. "
|
||||
f"Actions executed: {executed_actions}"
|
||||
)
|
||||
|
||||
|
||||
# If the bug was present, it would only try 'tap reels tab' 2 times, mask it forever,
|
||||
# and then spam 'force start instagram' for the remaining steps.
|
||||
# With the fix, it tries 2 times, restarts, tries 2 times, restarts, etc.
|
||||
|
||||
@@ -13,20 +13,15 @@ Design Philosophy:
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pkgutil
|
||||
import sys
|
||||
from argparse import Namespace
|
||||
from datetime import datetime, timedelta
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.session_state import SessionState, SessionStateEncoder
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# CONTRACT 1: SessionState Serialization is ALWAYS Safe
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -61,10 +56,9 @@ def _make_session_with_args(**extra_args):
|
||||
for k, v in extra_args.items():
|
||||
setattr(base_args, k, v)
|
||||
|
||||
class FakeConfig:
|
||||
pass
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
configs = FakeConfig()
|
||||
configs = Config(first_run=True)
|
||||
configs.args = base_args
|
||||
return SessionState(configs)
|
||||
|
||||
@@ -144,8 +138,6 @@ class TestPersistenceContract:
|
||||
"""
|
||||
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
|
||||
|
||||
from GramAddict.core.persistent_list import PersistentList
|
||||
|
||||
out_file = tmp_path / "test_sessions.json"
|
||||
|
||||
# Create a session with limits set (as bot_flow.py does)
|
||||
@@ -213,10 +205,7 @@ class TestProductionParityContract:
|
||||
# Known, audited divergence points. Any NEW divergence must be added here
|
||||
# with a justification, or the test fails.
|
||||
KNOWN_DIVERGENCES = {
|
||||
# (file_basename, line_number): "justification"
|
||||
("persistent_list.py", 30): "Prevents test side-effects on disk. Covered by TestPersistenceContract.",
|
||||
("bot_flow.py", 96): "check_production_integrity: validates no MagicMocks in prod. Not needed in tests.",
|
||||
("config.py", 22): "Prevents pytest args from being parsed by configargparse. Structural necessity.",
|
||||
# No known divergences. 100% production parity achieved.
|
||||
}
|
||||
|
||||
def test_all_test_divergence_points_are_audited(self):
|
||||
@@ -244,8 +233,8 @@ class TestProductionParityContract:
|
||||
unaudited.append(f"{fname}:{lineno} → {line.strip()}")
|
||||
|
||||
assert not unaudited, (
|
||||
f"UNAUDITED TEST DIVERGENCE DETECTED!\n"
|
||||
f"The following production code paths behave differently in test vs production:\n"
|
||||
"UNAUDITED TEST DIVERGENCE DETECTED!\n"
|
||||
"The following production code paths behave differently in test vs production:\n"
|
||||
+ "\n".join(f" ❌ {u}" for u in unaudited)
|
||||
+ "\n\nEach divergence is a potential 'lying test'. "
|
||||
"Add it to KNOWN_DIVERGENCES with a justification, or remove the guard."
|
||||
@@ -303,7 +292,6 @@ class TestImportIntegrityContract:
|
||||
except Exception as e:
|
||||
errors.append(f"{mod_path}: {type(e).__name__}: {e}")
|
||||
|
||||
assert not errors, (
|
||||
f"MODULE IMPORT FAILURES — The bot would crash on startup!\n"
|
||||
+ "\n".join(f" ❌ {e}" for e in errors)
|
||||
assert not errors, "MODULE IMPORT FAILURES — The bot would crash on startup!\n" + "\n".join(
|
||||
f" ❌ {e}" for e in errors
|
||||
)
|
||||
|
||||
@@ -28,36 +28,45 @@ from GramAddict.core.perception.action_memory import (
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Fake UIMemoryDB — replaces MagicMock to satisfy mock ban
|
||||
# Spy UIMemoryDB — wraps the REAL UIMemoryDB to track calls
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class FakeUIMemoryDB:
|
||||
class SpyUIMemoryDB:
|
||||
"""
|
||||
Real fake implementation of UIMemoryDB that tracks method calls
|
||||
without using unittest.mock. Satisfies the project's strict mock ban.
|
||||
Wraps the real UIMemoryDB to track method calls for assertions
|
||||
while exercising the actual production code path.
|
||||
|
||||
When Qdrant is down, UIMemoryDB gracefully degrades (client=None)
|
||||
and all operations become no-ops — which is the exact production
|
||||
behavior we want to test against.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
from GramAddict.core.qdrant_memory import UIMemoryDB
|
||||
|
||||
self._real = UIMemoryDB()
|
||||
self.store_memory_calls = []
|
||||
self.boost_confidence_calls = []
|
||||
self.decay_confidence_calls = []
|
||||
self.retrieve_memory_calls = []
|
||||
|
||||
def retrieve_memory(self, intent, xml_context):
|
||||
def retrieve_memory(self, intent, xml_context, **kwargs):
|
||||
self.retrieve_memory_calls.append((intent, xml_context))
|
||||
return None
|
||||
return self._real.retrieve_memory(intent, xml_context, **kwargs)
|
||||
|
||||
def store_memory(self, intent, xml_context, node_dict):
|
||||
self.store_memory_calls.append((intent, xml_context, node_dict))
|
||||
self._real.store_memory(intent, xml_context, node_dict)
|
||||
|
||||
def boost_confidence(self, intent, xml_context):
|
||||
def boost_confidence(self, intent, xml_context=None, **kwargs):
|
||||
self.boost_confidence_calls.append((intent, xml_context))
|
||||
self._real.boost_confidence(intent, xml_context, **kwargs)
|
||||
|
||||
def decay_confidence(self, intent, xml_context):
|
||||
def decay_confidence(self, intent, xml_context=None, **kwargs):
|
||||
self.decay_confidence_calls.append((intent, xml_context))
|
||||
self._real.decay_confidence(intent, xml_context, **kwargs)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -166,9 +175,7 @@ class TestVLMResponseParsing:
|
||||
)
|
||||
def test_parse_yes_no(self, response, expected):
|
||||
result = _parse_yes_no(response)
|
||||
assert result is expected, (
|
||||
f"_parse_yes_no('{response}') returned {result}, expected {expected}"
|
||||
)
|
||||
assert result is expected, f"_parse_yes_no('{response}') returned {result}, expected {expected}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -266,9 +273,7 @@ class TestStructuralGuards:
|
||||
bounds=(216, 2300, 432, 2400),
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "tap explore tab", screen_height=2400
|
||||
)
|
||||
filtered = self.resolver.filter_navigation_conflicts(candidates, "tap explore tab", screen_height=2400)
|
||||
assert len(filtered) == 1
|
||||
assert filtered[0].center_y == 2350
|
||||
|
||||
@@ -289,9 +294,7 @@ class TestStructuralGuards:
|
||||
center_y=2350,
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "tap post author username", screen_height=2400
|
||||
)
|
||||
filtered = self.resolver.filter_navigation_conflicts(candidates, "tap post author username", screen_height=2400)
|
||||
author_nodes = [n for n in filtered if n.text == "photographer_jane"]
|
||||
assert len(author_nodes) == 1
|
||||
|
||||
@@ -323,7 +326,8 @@ class TestStructuralGuards:
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "post author username text (exclude bottom tabs)",
|
||||
candidates,
|
||||
"post author username text (exclude bottom tabs)",
|
||||
screen_height=2400,
|
||||
)
|
||||
# The author username at y=2012 MUST survive
|
||||
@@ -462,8 +466,7 @@ class TestSemanticMatchGuard:
|
||||
def test_intent_matches_node(self, intent, semantic_string, expected):
|
||||
result = _intent_matches_node(intent, semantic_string)
|
||||
assert result is expected, (
|
||||
f"_intent_matches_node('{intent}', '{semantic_string[:50]}...') "
|
||||
f"returned {result}, expected {expected}"
|
||||
f"_intent_matches_node('{intent}', '{semantic_string[:50]}...') " f"returned {result}, expected {expected}"
|
||||
)
|
||||
|
||||
|
||||
@@ -482,9 +485,9 @@ class TestActionMemoryLifecycle:
|
||||
"""Tests the full click tracking → confirmation/rejection lifecycle."""
|
||||
|
||||
def _make_memory(self):
|
||||
"""Create ActionMemory with a FakeUIMemoryDB."""
|
||||
fake_db = FakeUIMemoryDB()
|
||||
return ActionMemory(ui_memory=fake_db), fake_db
|
||||
"""Create ActionMemory with a SpyUIMemoryDB wrapping the real UIMemoryDB."""
|
||||
spy_db = SpyUIMemoryDB()
|
||||
return ActionMemory(ui_memory=spy_db), spy_db
|
||||
|
||||
def test_confirm_correct_follow_stores_in_memory(self):
|
||||
"""A confirmed 'follow' click on a Follow button must be stored."""
|
||||
@@ -539,9 +542,7 @@ class TestActionMemoryLifecycle:
|
||||
memory, _ = self._make_memory()
|
||||
|
||||
post_xml_success = '<node text="Following" />'
|
||||
result = memory.verify_success(
|
||||
"follow", pre_click_xml="<node/>", post_click_xml=post_xml_success
|
||||
)
|
||||
result = memory.verify_success("follow", pre_click_xml="<node/>", post_click_xml=post_xml_success)
|
||||
assert result is True
|
||||
|
||||
def test_verify_view_post_success(self):
|
||||
@@ -549,9 +550,7 @@ class TestActionMemoryLifecycle:
|
||||
memory, _ = self._make_memory()
|
||||
|
||||
post_xml_success = '<node resource-id="com.instagram.android:id/row_feed_button_like" />'
|
||||
result = memory.verify_success(
|
||||
"view a post", pre_click_xml="<node/>", post_click_xml=post_xml_success
|
||||
)
|
||||
result = memory.verify_success("view a post", pre_click_xml="<node/>", post_click_xml=post_xml_success)
|
||||
assert result is True
|
||||
|
||||
|
||||
@@ -828,8 +827,8 @@ class TestVerifySuccessStructuralDelta:
|
||||
"""Tests the structural XML diff logic in verify_success."""
|
||||
|
||||
def _make_memory(self):
|
||||
fake_db = FakeUIMemoryDB()
|
||||
return ActionMemory(ui_memory=fake_db), fake_db
|
||||
spy_db = SpyUIMemoryDB()
|
||||
return ActionMemory(ui_memory=spy_db), spy_db
|
||||
|
||||
def test_toggle_massive_shift_is_navigation_error(self):
|
||||
"""
|
||||
@@ -917,4 +916,3 @@ class TestVerifySuccessStructuralDelta:
|
||||
|
||||
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
|
||||
assert result is False
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ without relying on brittle mock LLM responses.
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
from tests.e2e.conftest import make_real_device_with_xml
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -21,14 +20,7 @@ def sae(make_real_device_with_xml):
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
db = ScreenMemoryDB()
|
||||
if db.is_connected:
|
||||
try:
|
||||
db.client.delete(
|
||||
collection_name=db.collection_name,
|
||||
points_selector={"filter": {}}, # Delete all points
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
db.wipe_collection()
|
||||
return engine
|
||||
|
||||
|
||||
@@ -148,3 +140,44 @@ class TestSAELoop:
|
||||
|
||||
foreign_xml = '<node package="com.apple.ios" />'
|
||||
assert sae.is_instagram_foreground(xml_dump=foreign_xml) is False
|
||||
|
||||
def test_false_positive_modal_infinite_loop_trap(self, sae, monkeypatch):
|
||||
"""
|
||||
Reproduces a production trap where ScreenIdentity overrides Qdrant for structural
|
||||
markers (Priority 0). If the LLM identifies the modal as a false positive, it unlearns
|
||||
it in Qdrant. However, without the fix, the next time GOAP evaluates the state,
|
||||
ScreenIdentity STILL says MODAL because of Priority 0, triggering an infinite loop.
|
||||
This test ensures ScreenIdentity respects NORMAL memory overrides.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
# Load real home feed XML
|
||||
xml_dump = Path("tests/e2e/fixtures/home_feed_real.xml").read_text()
|
||||
|
||||
# Inject the Priority 0 structural marker
|
||||
xml_dump = xml_dump.replace(
|
||||
'<node index="0" text="" resource-id="android:id/content"',
|
||||
'<node resource-id="com.instagram.android:id/gallery_cancel_button" bounds="[0,0][100,100]" />\n<node index="0" text="" resource-id="android:id/content"',
|
||||
)
|
||||
|
||||
sae.device.deviceV2.xml = xml_dump
|
||||
sae.device.deviceV2.info["screenOn"] = True
|
||||
|
||||
# Simulate LLM unlearning by storing this exact state as NORMAL
|
||||
compressed = sae._compress_xml(xml_dump)
|
||||
ScreenMemoryDB().store_screen(compressed, "NORMAL")
|
||||
|
||||
identity = ScreenIdentity("testuser")
|
||||
# Ensure we inject the device so get_screenshot_b64 doesn't crash if it falls back
|
||||
identity.device = sae.device
|
||||
|
||||
# The bug: this would return MODAL because of Priority 0, ignoring the DB
|
||||
# The fix: it should return HOME_FEED because is_normal_override = True skips the MODAL check
|
||||
result = identity.identify(xml_dump)
|
||||
|
||||
assert (
|
||||
result["screen_type"].value == "home_feed"
|
||||
), f"Infinite loop trap: Expected home_feed, got {result['screen_type'].value}"
|
||||
|
||||
@@ -70,3 +70,31 @@ def test_permission_dialog_terminates_chain(make_real_device_with_xml, e2e_workf
|
||||
|
||||
# BACK must have been pressed
|
||||
assert "back" in device.pressed_keys, "obstacle_guard did not press BACK — the dialog stays on screen!"
|
||||
|
||||
|
||||
def test_sae_escapes_permission_dialog_via_vlm(make_real_device_with_xml):
|
||||
"""
|
||||
Ensures that the SituationalAwarenessEngine's ensure_clear_screen loop
|
||||
can correctly use the VLM to escape an OBSTACLE_SYSTEM dialog.
|
||||
"""
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
# First XML is the permission dialog, second is the normal feed.
|
||||
device = make_real_device_with_xml([PERMISSION_DIALOG_XML, NORMAL_POST_XML])
|
||||
sae = SituationalAwarenessEngine.get_instance(device)
|
||||
|
||||
# We must wipe Qdrant memory for this situation so it forces an LLM call.
|
||||
_ = sae._compress_xml(PERMISSION_DIALOG_XML)
|
||||
sae.episodes.recall = lambda x: None # Force LLM instead of recalled memory for test determinism
|
||||
|
||||
# Ensure clear screen MUST return True, meaning it successfully cleared the obstacle.
|
||||
success = sae.ensure_clear_screen(max_attempts=3, initial_xml=PERMISSION_DIALOG_XML)
|
||||
|
||||
assert success is True, "ensure_clear_screen failed to escape the system dialog!"
|
||||
|
||||
# Verify that the LLM decided to either click the 'Don't allow' button or press BACK.
|
||||
# The 'Don't allow' button is at [100,1240][980,1340], center is (540, 1290).
|
||||
clicked_deny = any(abs(x - 540) < 50 and abs(y - 1290) < 50 for x, y in device.clicks)
|
||||
pressed_back = "back" in device.pressed_keys
|
||||
|
||||
assert clicked_deny or pressed_back, "VLM failed to click the 'Don't allow' button or press back!"
|
||||
|
||||
Reference in New Issue
Block a user