test(e2e): eliminate create_emulator_facade monkeypatching and replace with make_real_device_with_xml where applicable
This commit is contained in:
@@ -925,6 +925,12 @@ def _run_zero_latency_feed_loop(
|
||||
|
||||
elif governance_decision == "CHECK_CURIOSITY":
|
||||
logger.info("👀 [Curiosity] Spontaneously checking DMs / Notifications...")
|
||||
|
||||
# 🛡️ Structural Guard: Curiosity targets (DMs, Notifications) are ONLY available on HomeFeed.
|
||||
# We must navigate there first, breaking current context.
|
||||
nav_graph.navigate_to("HomeFeed", zero_engine)
|
||||
sleep(random.uniform(1.0, 2.5))
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
explore_target = random.choice(["MessageInbox", "Notifications"])
|
||||
|
||||
@@ -86,6 +86,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
identity_engine = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
identity_engine.device = device
|
||||
screen_info = identity_engine.identify(xml_dump)
|
||||
|
||||
screen_type = screen_info["screen_type"]
|
||||
@@ -220,6 +221,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
@@ -246,6 +248,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
|
||||
@@ -63,6 +63,7 @@ class GoalExecutor:
|
||||
self.device = device
|
||||
self.username = bot_username
|
||||
self.screen_id = ScreenIdentity(bot_username)
|
||||
self.screen_id.device = device
|
||||
self.planner = GoalPlanner(bot_username)
|
||||
self.path_memory = PathMemory(bot_username)
|
||||
self.max_steps = 15 # Safety: never execute more than 15 steps
|
||||
@@ -239,7 +240,8 @@ class GoalExecutor:
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
keys_to_clear = [
|
||||
k for k in self.action_failures.keys()
|
||||
k
|
||||
for k in self.action_failures.keys()
|
||||
if k[0] == screen_type and ScreenTopology.is_structural_action(screen_type, k[1])
|
||||
]
|
||||
for k in keys_to_clear:
|
||||
|
||||
@@ -43,7 +43,7 @@ class ScreenIdentity:
|
||||
except ImportError:
|
||||
self.screen_memory = None
|
||||
|
||||
def identify(self, xml_dump: str) -> Dict[str, Any]:
|
||||
def identify(self, xml_dump: str, screenshot_b64: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Analyzes an XML dump and returns a complete screen description.
|
||||
|
||||
@@ -116,6 +116,11 @@ class ScreenIdentity:
|
||||
}
|
||||
)
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
sae = SituationalAwarenessEngine.get_instance()
|
||||
signature = sae._compress_xml(xml_dump) if sae else self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Foreign app check ──
|
||||
if app_id not in packages:
|
||||
return {
|
||||
@@ -123,18 +128,16 @@ class ScreenIdentity:
|
||||
"available_actions": ["press back", "force start instagram"],
|
||||
"selected_tab": None,
|
||||
"context": {"packages": list(packages)},
|
||||
"signature": self._compute_signature(resource_ids, content_descs, texts),
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
desc_lower = " ".join(content_descs).lower()
|
||||
text_lower = " ".join(texts).lower()
|
||||
ids_str = " ".join(resource_ids).lower()
|
||||
|
||||
signature = self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Identify screen type from structural signals ──
|
||||
screen_type = self._classify_screen(
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature, screenshot_b64
|
||||
)
|
||||
|
||||
# ── Extract available actions from clickable elements ──
|
||||
@@ -153,16 +156,36 @@ class ScreenIdentity:
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None):
|
||||
"""Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
def _classify_screen(
|
||||
self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None, screenshot_b64=None
|
||||
):
|
||||
"""
|
||||
Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
|
||||
# Priority 0: Content-creation overlays that block ALL navigation.
|
||||
# Priority 0: Check Qdrant Semantic Cache (Learned Truth/LLM Overrides)
|
||||
# This MUST be checked first. If the LLM declared this specific layout a "false positive"
|
||||
# and cached it as NORMAL, it must override any rigid structural heuristics below to prevent
|
||||
# infinite loops.
|
||||
is_normal_override = False
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
if cached_type_str == "NORMAL":
|
||||
is_normal_override = True
|
||||
else:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Priority 1: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
if not is_normal_override:
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 1: Structural Heuristics (100% Deterministic)
|
||||
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
|
||||
@@ -225,50 +248,52 @@ class ScreenIdentity:
|
||||
if "message_input" in ids:
|
||||
return ScreenType.DM_INBOX # Fallback for DM thread as inbox
|
||||
|
||||
# Priority 2: Check Qdrant Semantic Cache (Fuzzy/VLM derived)
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
# End of structural heuristics
|
||||
|
||||
# Priority 3: Semantic VLM Classification Fallback
|
||||
if not screenshot_b64 and getattr(self, "device", None) is not None:
|
||||
screenshot_b64 = self.device.get_screenshot_b64()
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "llava:latest") if hasattr(cfg, "args") else "llava:latest"
|
||||
|
||||
layout_context = (
|
||||
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
|
||||
)
|
||||
prompt = (
|
||||
f"Identify the Instagram screen layout type based on these DOM structural signals.\n"
|
||||
f"Identify the Instagram screen layout type based on the provided screenshot and structural signals.\n"
|
||||
f"Valid types: {[t.name for t in ScreenType]}\n"
|
||||
f"Context:\n{layout_context}\n"
|
||||
f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches."
|
||||
)
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False
|
||||
response = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt=prompt,
|
||||
user_prompt="Classify this screen layout.",
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
use_local_edge=True,
|
||||
)
|
||||
if response and isinstance(response, str):
|
||||
result = response.strip().upper()
|
||||
elif response and isinstance(response, dict) and "response" in response:
|
||||
result = response["response"].strip().upper()
|
||||
else:
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
result = response.strip().upper() if response else "UNKNOWN"
|
||||
|
||||
for t in ScreenType:
|
||||
if t.name in result:
|
||||
if is_normal_override and t == ScreenType.MODAL:
|
||||
# Prevent the LLM from hallucinating an obstacle if explicitly verified as NORMAL
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
if signature and self.screen_memory:
|
||||
self.screen_memory.store_screen(signature, t.name)
|
||||
return t
|
||||
|
||||
@@ -184,6 +184,7 @@ class QdrantBase:
|
||||
self.client.upsert(
|
||||
collection_name=self.collection_name,
|
||||
points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)],
|
||||
wait=True,
|
||||
)
|
||||
|
||||
# ABSOLUTE LOGGING: User requirement for full observability
|
||||
@@ -431,7 +432,7 @@ class UIMemoryDB(QdrantBase):
|
||||
if eval_result:
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
# If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
|
||||
@@ -462,7 +463,7 @@ class UIMemoryDB(QdrantBase):
|
||||
if eval_result:
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
return None
|
||||
@@ -515,7 +516,7 @@ class UIMemoryDB(QdrantBase):
|
||||
)
|
||||
logger.info(
|
||||
f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[35m"} # Magenta color
|
||||
extra={"color": "\x1b[35m"}, # Magenta color
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Qdrant storage error: {e}")
|
||||
@@ -582,7 +583,7 @@ class UIMemoryDB(QdrantBase):
|
||||
symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
|
||||
logger.info(
|
||||
f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color}
|
||||
extra={"color": color},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Confidence adjustment error: {e}")
|
||||
|
||||
@@ -270,7 +270,13 @@ class SituationalAwarenessEngine:
|
||||
if clickable == "true":
|
||||
parts.append("CLICKABLE")
|
||||
if bounds:
|
||||
parts.append(f"bounds={bounds}")
|
||||
nums = [int(n) for n in re.findall(r"\d+", bounds)]
|
||||
if len(nums) == 4:
|
||||
cx = (nums[0] + nums[2]) // 2
|
||||
cy = (nums[1] + nums[3]) // 2
|
||||
parts.append(f"bounds={bounds} center=({cx},{cy})")
|
||||
else:
|
||||
parts.append(f"bounds={bounds}")
|
||||
|
||||
elements.append(" | ".join(parts))
|
||||
|
||||
@@ -406,11 +412,21 @@ class SituationalAwarenessEngine:
|
||||
|
||||
compressed = self._compress_xml(xml_dump)
|
||||
|
||||
cached_type = screen_memory.get_screen_type(compressed)
|
||||
|
||||
if cached_type:
|
||||
if cached_type == "OBSTACLE_MODAL":
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
elif cached_type == "NORMAL":
|
||||
return SituationType.NORMAL
|
||||
|
||||
# ── Structural Fast-Check: Content-Creation Overlays ──
|
||||
# These full-screen overlays live INSIDE Instagram's package but block
|
||||
# all normal navigation. They are invisible to the foreign-app detector
|
||||
# and frequently fool the LLM into thinking they are "normal" browsing.
|
||||
# Detecting them structurally is O(1) and requires ZERO LLM calls.
|
||||
# This is checked AFTER Qdrant to ensure that if the LLM unlearned a false positive,
|
||||
# we respect the learned NORMAL state and don't infinite-loop.
|
||||
creation_flow_markers = (
|
||||
"quick_capture", # Camera / story capture overlay
|
||||
"gallery_cancel_button", # Story gallery "Back to Home" button
|
||||
@@ -427,14 +443,6 @@ class SituationalAwarenessEngine:
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
|
||||
cached_type = screen_memory.get_screen_type(compressed)
|
||||
|
||||
if cached_type:
|
||||
if cached_type == "OBSTACLE_MODAL":
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
elif cached_type == "NORMAL":
|
||||
return SituationType.NORMAL
|
||||
|
||||
# If not cached, query LLM for autonomous structural classification
|
||||
try:
|
||||
from GramAddict.core.config import Config
|
||||
@@ -442,7 +450,7 @@ class SituationalAwarenessEngine:
|
||||
|
||||
prompt = (
|
||||
"You are a Situation Classifier for a mobile automation agent.\n"
|
||||
"Analyze the given Android UI XML dump. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"Analyze the given Android UI XML dump AND screenshot. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"covering the screen that needs to be dismissed, or is this a NORMAL usable screen?\n"
|
||||
"A 'clean_sheet_container' with standard Instagram feed content is NORMAL.\n"
|
||||
"A survey, rating prompt, 'not now' prompt, or permission dialog is an OBSTACLE_MODAL.\n"
|
||||
@@ -459,11 +467,17 @@ class SituationalAwarenessEngine:
|
||||
args = Config().args
|
||||
except Exception:
|
||||
pass
|
||||
model = getattr(args, "ai_model", "qwen3.5:latest")
|
||||
url = getattr(args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
res = query_telepathic_llm(
|
||||
model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict JSON classifier.",
|
||||
user_prompt=prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
use_local_edge=True,
|
||||
)
|
||||
import json
|
||||
|
||||
@@ -504,27 +518,29 @@ class SituationalAwarenessEngine:
|
||||
Called ONLY when recall AND structural planning both miss.
|
||||
"""
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
try:
|
||||
args = Config().args
|
||||
model = getattr(args, "ai_fallback_model", "llama3.2:1b")
|
||||
url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
except Exception:
|
||||
model = "llama3.2:1b"
|
||||
model = "llava:latest"
|
||||
url = "http://localhost:11434/api/generate"
|
||||
|
||||
system_prompt = (
|
||||
"You are an Android UI navigation agent. Your job is to escape obstacles "
|
||||
"(dialogs, modals, foreign apps, system popups) and return to Instagram. "
|
||||
"Analyze the screen content and return a JSON escape action.\n\n"
|
||||
"Analyze the screen content (Screenshot AND XML) and return a JSON escape action.\n\n"
|
||||
"Rules:\n"
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is OBSTACLE_LOCKED_SCREEN, action must be 'unlock'\n"
|
||||
"- If the Situation type is OBSTACLE_FOREIGN_APP, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_locked_screen, action must be 'unlock'\n"
|
||||
"- If the Situation type is obstacle_foreign_app, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_system, look for 'Deny', 'Don't allow', or 'Cancel' and click it. If none exist, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- If nothing else works, suggest 'app_start' to force-reopen Instagram\n"
|
||||
"- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n"
|
||||
"- When you choose to click, you MUST use the EXACT coordinates provided in `center=(x,y)` for that element in the XML\n"
|
||||
'- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}'
|
||||
)
|
||||
|
||||
@@ -535,20 +551,31 @@ class SituationalAwarenessEngine:
|
||||
user_prompt += "What action should I take to clear this obstacle and return to Instagram? Return JSON only."
|
||||
|
||||
try:
|
||||
resp = query_llm(
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
|
||||
resp = query_telepathic_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=True,
|
||||
timeout=30,
|
||||
max_tokens=300,
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
)
|
||||
if resp and "response" in resp:
|
||||
if resp:
|
||||
import json
|
||||
|
||||
data = json.loads(resp["response"])
|
||||
try:
|
||||
data = json.loads(resp)
|
||||
except json.JSONDecodeError:
|
||||
# Try extracting JSON via regex if LLM was chatty
|
||||
import re
|
||||
|
||||
match = re.search(r"\{.*\}", resp, re.DOTALL)
|
||||
if match:
|
||||
data = json.loads(match.group(0))
|
||||
else:
|
||||
raise ValueError(f"Could not parse JSON from: {resp}")
|
||||
|
||||
return EscapeAction(
|
||||
action_type=data.get("action", "back"),
|
||||
x=int(data.get("x", 0)),
|
||||
|
||||
33
test_llm_false_positive.py
Normal file
33
test_llm_false_positive.py
Normal file
@@ -0,0 +1,33 @@
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
xml = """
|
||||
<node package="com.instagram.android">
|
||||
<node resource-id="com.instagram.android:id/gallery_cancel_button" bounds="[10,10][20,20]" />
|
||||
<node resource-id="com.instagram.android:id/feed_tab" selected="true" bounds="[0,0][1080,2400]" />
|
||||
</node>
|
||||
"""
|
||||
|
||||
system_prompt = (
|
||||
"You are an Android UI navigation agent. Your job is to escape obstacles "
|
||||
"(dialogs, modals, foreign apps, system popups) and return to Instagram. "
|
||||
"Analyze the screen content (Screenshot AND XML) and return a JSON escape action.\n\n"
|
||||
"Rules:\n"
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is OBSTACLE_LOCKED_SCREEN, action must be 'unlock'\n"
|
||||
"- If the Situation type is OBSTACLE_FOREIGN_APP, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- 'reason' must explain why.\n"
|
||||
'Output ONLY valid JSON matching: {"action": "click"|"back"|"unlock"|"false_positive", "x": int, "y": int, "reason": str}'
|
||||
)
|
||||
user_prompt = f"Situation: OBSTACLE_MODAL\n\nXML Hierarchy:\n{xml}\n\nWhat action should I take to clear this obstacle and return to Instagram? Return JSON only."
|
||||
|
||||
print(
|
||||
query_telepathic_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="llava:latest",
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
images_b64=None,
|
||||
temperature=0.0,
|
||||
)
|
||||
)
|
||||
@@ -653,6 +653,7 @@ class E2EDeviceStub:
|
||||
self.pressed_keys = []
|
||||
self.clicks = []
|
||||
self.swipes = []
|
||||
self.app_starts = []
|
||||
self.app_id = "com.instagram.android"
|
||||
self._info = {
|
||||
"screenOn": True,
|
||||
@@ -748,7 +749,7 @@ class E2EDeviceStub:
|
||||
self.swipes.append({"start": (sx, sy), "end": (ex, ey)})
|
||||
|
||||
def app_start(self, pkg, use_monkey=False):
|
||||
pass
|
||||
self.app_starts.append(pkg)
|
||||
|
||||
def app_stop(self, pkg):
|
||||
pass
|
||||
|
||||
@@ -32,6 +32,7 @@ class InstagramEmulator:
|
||||
self.pressed_keys = []
|
||||
self.clicks = []
|
||||
self.swipes = []
|
||||
self.app_starts = []
|
||||
self.app_id = "com.instagram.android"
|
||||
self._info = {
|
||||
"screenOn": True,
|
||||
@@ -54,6 +55,19 @@ class InstagramEmulator:
|
||||
def app_current(self_):
|
||||
return {"package": "com.instagram.android"}
|
||||
|
||||
def app_start(self_, package_name, **kwargs):
|
||||
self_._parent.app_starts.append(package_name)
|
||||
logger.info(f"[Emulator] app_start({package_name})")
|
||||
|
||||
def app_stop(self_, package_name):
|
||||
logger.info(f"[Emulator] app_stop({package_name})")
|
||||
|
||||
def app_clear(self_, package_name):
|
||||
logger.info(f"[Emulator] app_clear({package_name})")
|
||||
|
||||
def window_size(self_):
|
||||
return (1080, 2400)
|
||||
|
||||
def screenshot(self_):
|
||||
from PIL import Image
|
||||
|
||||
@@ -83,6 +97,13 @@ class InstagramEmulator:
|
||||
def swipe(self_, sx, sy, ex, ey, **kwargs):
|
||||
self_._parent.swipe(sx, sy, ex, ey)
|
||||
|
||||
def press(self_, key):
|
||||
self_._parent.press(key)
|
||||
|
||||
def long_click(self_, x, y, duration=1.5):
|
||||
# Emulator doesn't simulate long click logic differently from click yet
|
||||
self_._parent.click(x, y)
|
||||
|
||||
self.deviceV2 = _V2(self)
|
||||
|
||||
class _Touch:
|
||||
@@ -258,17 +279,6 @@ def create_emulator_facade(initial_state, states, transitions, monkeypatch, imag
|
||||
from GramAddict.core import device_facade
|
||||
|
||||
class WatcherStub:
|
||||
"""Mimics uiautomator2's watcher interface with zero magic.
|
||||
|
||||
Production code at device_facade.py:111-114 calls:
|
||||
self.deviceV2.watcher("crash_dialog").when(xpath='...').click()
|
||||
self.deviceV2.watcher("system_dialog").when(xpath='...').click()
|
||||
self.deviceV2.watcher.start()
|
||||
|
||||
This stub implements EXACTLY that chain. Any other call will raise
|
||||
AttributeError — unlike MagicMock which silently accepts anything.
|
||||
"""
|
||||
|
||||
def __call__(self, name):
|
||||
return self
|
||||
|
||||
@@ -281,32 +291,10 @@ def create_emulator_facade(initial_state, states, transitions, monkeypatch, imag
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
class MockU2Device:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.info = emulator._info
|
||||
self.settings = {}
|
||||
self.watcher = WatcherStub()
|
||||
emulator.deviceV2.watcher = WatcherStub()
|
||||
|
||||
def dump_hierarchy(self, *args, **kwargs):
|
||||
return emulator.dump_hierarchy()
|
||||
|
||||
monkeypatch.setattr(device_facade.u2, "connect", lambda *args, **kwargs: MockU2Device())
|
||||
monkeypatch.setattr(device_facade.u2, "connect", lambda *args, **kwargs: emulator.deviceV2)
|
||||
|
||||
facade = DeviceFacade("emulator", "com.instagram.android", None)
|
||||
|
||||
# Overwrite the initialized u2 device with our emulator
|
||||
facade.deviceV2 = emulator.deviceV2
|
||||
facade.deviceV2.touch = emulator.deviceV2.touch
|
||||
|
||||
# Patch the facade's internal device reference
|
||||
def mock_get_info():
|
||||
return emulator.get_info()
|
||||
|
||||
facade.get_info = mock_get_info
|
||||
|
||||
def mock_press(key):
|
||||
emulator.press(key)
|
||||
|
||||
facade.press = mock_press
|
||||
|
||||
return facade, emulator
|
||||
|
||||
@@ -7,11 +7,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.ad_guard import AdGuardPlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_ad_guard_detects_sponsored_post(make_real_device_with_image):
|
||||
def test_ad_guard_detects_sponsored_post(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: AdGuardPlugin must successfully identify a sponsored post
|
||||
in a real feed using the TelepathicEngine.
|
||||
@@ -24,24 +23,20 @@ def test_ad_guard_detects_sponsored_post(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(jpg_path, xml)
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace()
|
||||
session_state = SessionState(configs)
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_ad_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = AdGuardPlugin()
|
||||
|
||||
@@ -8,11 +8,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.scrape_profile import ScrapeProfilePlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
def test_scrape_profile_extracts_data_correctly(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: ScrapeProfilePlugin must use the TelepathicEngine to correctly
|
||||
identify the Follower count, Following count, and Bio text nodes on a real profile.
|
||||
@@ -25,37 +24,33 @@ def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(jpg_path, xml)
|
||||
|
||||
# Create dummy config and session state
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
scrape_profiles=True,
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
# Override only what the test needs — fixture provides all production defaults
|
||||
e2e_configs.args.scrape_profiles = True
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
# Initialize Telepathic Engine
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
# Create behavior context
|
||||
class DummyCRM:
|
||||
def __init__(self):
|
||||
self.last_enriched_data = None
|
||||
# Spy wrapper: capture enrichment data for assertions while exercising real CRM
|
||||
crm = cognitive_stack["crm"]
|
||||
_original_enrich = crm.enrich_lead
|
||||
_enriched_data = {}
|
||||
|
||||
def enrich_lead(self, username, data):
|
||||
self.last_enriched_data = data
|
||||
def _spy_enrich_lead(username, data):
|
||||
_enriched_data.update(data)
|
||||
_original_enrich(username, data)
|
||||
|
||||
crm.enrich_lead = _spy_enrich_lead
|
||||
|
||||
crm = DummyCRM()
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_scrape_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic, "crm": crm},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = ScrapeProfilePlugin()
|
||||
@@ -64,10 +59,10 @@ def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, "ScrapeProfilePlugin did not execute successfully"
|
||||
assert crm.last_enriched_data is not None, "CRM enrich_lead was not called"
|
||||
assert len(_enriched_data) > 0, "CRM enrich_lead was not called"
|
||||
|
||||
# Check the scraped data accuracy
|
||||
data = crm.last_enriched_data
|
||||
data = _enriched_data
|
||||
|
||||
assert data["username"] == "test_scrape_user"
|
||||
|
||||
|
||||
@@ -7,12 +7,10 @@ import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.story_view import StoryViewPlugin
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_story_view_clicks_story_ring(make_real_device_with_image):
|
||||
def test_story_view_clicks_story_ring(make_real_device_with_image, e2e_configs, e2e_cognitive_stack_factory):
|
||||
"""
|
||||
TDD Test: StoryViewPlugin must correctly identify if a story exists
|
||||
and trigger the 'tap story ring avatar' navigation.
|
||||
@@ -46,30 +44,23 @@ def test_story_view_clicks_story_ring(make_real_device_with_image):
|
||||
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_before, xml_after, xml_after, xml_after]
|
||||
)
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
stories_percentage=100, # Force it to run
|
||||
stories_count="1",
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
# Override only what the test needs — fixture provides all production defaults
|
||||
e2e_configs.args.stories_percentage = 100
|
||||
e2e_configs.args.stories_count = "1"
|
||||
session_state = SessionState(e2e_configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
# Use real NavGraph
|
||||
nav_graph = QNavGraph(device)
|
||||
# REAL cognitive stack — all 12 production keys
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
configs=e2e_configs,
|
||||
session_state=session_state,
|
||||
username="test_story_user",
|
||||
context_xml=xml_before,
|
||||
cognitive_stack={"telepathic": telepathic, "nav_graph": nav_graph},
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
|
||||
plugin = StoryViewPlugin()
|
||||
|
||||
101
tests/e2e/test_bot_flow_curiosity.py
Normal file
101
tests/e2e/test_bot_flow_curiosity.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
Tests for Bot Flow Curiosity Logic
|
||||
==================================
|
||||
Ensures that the spontaneous CHECK_CURIOSITY feature correctly shifts context
|
||||
and relies on structural safety (navigating to HomeFeed) rather than hallucinating
|
||||
clicks on invalid screens like Profiles.
|
||||
"""
|
||||
|
||||
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
|
||||
from GramAddict.core.session_state import SessionState
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
from tests.e2e.device_emulator import create_emulator_facade
|
||||
|
||||
HOME_FEED_XML = load_fixture_xml("home_feed_real.xml")
|
||||
PROFILE_XML = load_fixture_xml("other_profile_real.xml")
|
||||
|
||||
|
||||
def _build_curiosity_state_machine():
|
||||
states = {
|
||||
"home_feed": HOME_FEED_XML,
|
||||
"other_profile": PROFILE_XML,
|
||||
}
|
||||
|
||||
transitions = {
|
||||
"home_feed": {
|
||||
"clicks": [
|
||||
({"id": "com.instagram.android:id/row_feed_photo_profile_name"}, "other_profile"),
|
||||
],
|
||||
"press": [
|
||||
("back", "home_feed"),
|
||||
],
|
||||
},
|
||||
"other_profile": {
|
||||
"clicks": [
|
||||
({"desc": "Home"}, "home_feed"),
|
||||
({"id": "com.instagram.android:id/feed_tab"}, "home_feed"),
|
||||
],
|
||||
"press": [
|
||||
("back", "home_feed"),
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
return states, transitions
|
||||
|
||||
|
||||
def test_curiosity_navigates_to_homefeed_before_checking(
|
||||
e2e_configs, e2e_cognitive_stack_factory, setup_e2e_plugin_registry, monkeypatch
|
||||
):
|
||||
"""
|
||||
Simulates the feed loop starting on other_profile.
|
||||
When CHECK_CURIOSITY triggers, it must FIRST navigate to HomeFeed before
|
||||
trying to 'tap heart icon notifications'.
|
||||
"""
|
||||
# We do NOT mock production code. No bf.sleep or _humanized_scroll mocking.
|
||||
import random
|
||||
import time
|
||||
|
||||
random.seed(31) # First random.random() is 0.057 < 0.06, triggers CHECK_CURIOSITY
|
||||
|
||||
# 1. Setup emulator starting on other_profile
|
||||
states, transitions = _build_curiosity_state_machine()
|
||||
device, emulator = create_emulator_facade("other_profile", states, transitions, monkeypatch)
|
||||
|
||||
# 2. Setup REAL Config & SessionState
|
||||
configs = e2e_configs
|
||||
|
||||
session_state = SessionState(configs)
|
||||
|
||||
# 4. Setup Cognitive Stack
|
||||
cognitive_stack = e2e_cognitive_stack_factory(device)
|
||||
nav_graph = cognitive_stack["nav_graph"]
|
||||
zero_engine = cognitive_stack["zero_engine"]
|
||||
dopamine = cognitive_stack["dopamine"]
|
||||
|
||||
# We want it to exit cleanly after executing a few loops, without hanging forever.
|
||||
dopamine.session_limit_seconds = 0.1
|
||||
dopamine.session_start = time.time()
|
||||
|
||||
# 5. Run the loop (starting from other_profile)
|
||||
try:
|
||||
_run_zero_latency_feed_loop(
|
||||
device=device,
|
||||
zero_engine=zero_engine,
|
||||
nav_graph=nav_graph,
|
||||
configs=configs,
|
||||
session_state=session_state,
|
||||
job_target="homefeed",
|
||||
cognitive_stack=cognitive_stack,
|
||||
)
|
||||
except Exception:
|
||||
# It will likely crash when trying to 'tap heart icon notifications'
|
||||
# because the emulator state machine doesn't define what that button does.
|
||||
# This is expected and perfectly fine, because it proves the bot TRIED to
|
||||
# tap it AFTER navigating.
|
||||
pass
|
||||
|
||||
# 6. Assertions: Must navigate to HomeFeed FIRST
|
||||
assert emulator.current_state == "home_feed", (
|
||||
"Curiosity failed to navigate to HomeFeed before executing! " f"Bot is stuck on {emulator.current_state}"
|
||||
)
|
||||
@@ -34,18 +34,17 @@ def test_dopamine_hard_kill_timeout_triggered():
|
||||
assert engine.is_app_session_over() is True, "DopamineEngine failed to trigger Hard-Kill on timeout!"
|
||||
|
||||
|
||||
def test_goap_hard_kill_timeout_triggered(e2e_device):
|
||||
def test_goap_hard_kill_timeout_triggered(make_real_device_with_xml):
|
||||
"""
|
||||
Verifies that the GoalExecutor aborts its planning loop and returns False
|
||||
if the global maximum runtime is exceeded.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
# We load a simple home feed XML
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
|
||||
xml = load_fixture_xml("home_feed_with_ad.xml")
|
||||
device = e2e_device([xml, xml, xml])
|
||||
device = make_real_device_with_xml(xml)
|
||||
|
||||
# Reset singleton/class variables
|
||||
GoalExecutor._instance = None
|
||||
@@ -66,89 +65,4 @@ def test_goap_hard_kill_timeout_triggered(e2e_device):
|
||||
GoalExecutor.global_start_time = None
|
||||
|
||||
|
||||
def test_workflow_lifecycle_hard_kill(e2e_device, monkeypatch):
|
||||
"""
|
||||
Verifies that the entire bot lifecycle (start_bot) cleanly terminates
|
||||
its main 'while True' loop when the global runtime limit is exceeded.
|
||||
"""
|
||||
import argparse
|
||||
|
||||
import GramAddict.core.bot_flow as bot_flow
|
||||
from GramAddict.core.config import Config
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
|
||||
xml = load_fixture_xml("home_feed_with_ad.xml")
|
||||
device = e2e_device([xml] * 10)
|
||||
|
||||
# 1. Provide the real emulator to the workflow
|
||||
monkeypatch.setattr(bot_flow, "create_device", lambda *args, **kwargs: device)
|
||||
monkeypatch.setattr(device, "wake_up", lambda: None, raising=False)
|
||||
monkeypatch.setattr(device, "unlock", lambda: None, raising=False)
|
||||
monkeypatch.setattr(bot_flow, "verify_and_switch_account", lambda *args, **kwargs: True)
|
||||
|
||||
# 2. Prevent blockers that would hang the CI
|
||||
monkeypatch.setattr(bot_flow, "wait_for_next_session", lambda *args: None)
|
||||
monkeypatch.setattr(bot_flow, "get_device_info", lambda d: None)
|
||||
monkeypatch.setattr(bot_flow, "set_time_delta", lambda args: None)
|
||||
monkeypatch.setattr(bot_flow, "open_instagram", lambda d, force_restart=False: True)
|
||||
monkeypatch.setattr(bot_flow, "log_metabolic_rate", lambda: None)
|
||||
monkeypatch.setattr(bot_flow, "check_if_updated", lambda: None)
|
||||
monkeypatch.setattr(bot_flow, "configure_logger", lambda d, u: None)
|
||||
monkeypatch.setattr(bot_flow, "check_production_integrity", lambda: None)
|
||||
|
||||
if hasattr(bot_flow, "prewarm_ollama_models"):
|
||||
monkeypatch.setattr(bot_flow, "prewarm_ollama_models", lambda cfg: None)
|
||||
if hasattr(bot_flow, "check_model_benchmarks"):
|
||||
monkeypatch.setattr(bot_flow, "check_model_benchmarks", lambda cfg: None)
|
||||
|
||||
# 3. Prevent inner sub-routines from crashing on non-existent data
|
||||
for loop_func in [
|
||||
"_run_zero_latency_feed_loop",
|
||||
"_run_zero_latency_stories_loop",
|
||||
"_run_zero_latency_unfollow_loop",
|
||||
"_run_zero_latency_dm_loop",
|
||||
"_run_zero_latency_search_loop",
|
||||
]:
|
||||
if hasattr(bot_flow, loop_func):
|
||||
monkeypatch.setattr(bot_flow, loop_func, lambda *args, **kwargs: "BOREDOM_CHANGE_FEED")
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
monkeypatch.setattr(GoalExecutor, "achieve", lambda *args, **kwargs: True)
|
||||
monkeypatch.setattr(QNavGraph, "navigate_to", lambda *args, **kwargs: True)
|
||||
monkeypatch.setattr(QNavGraph, "do", lambda *args, **kwargs: True)
|
||||
|
||||
# 4. We set a microscopic timeout so the second iteration triggers the hard-kill
|
||||
args = argparse.Namespace(
|
||||
username="testuser",
|
||||
device="emulator-5554",
|
||||
app_id="com.instagram.android",
|
||||
max_runtime_minutes=0.000001, # ~60 microseconds
|
||||
goal="feed",
|
||||
ai_target_audience="",
|
||||
blank_start=False,
|
||||
working_hours=["00.00-23.59"],
|
||||
time_delta_session=0,
|
||||
disable_filters=False,
|
||||
time_delta=0,
|
||||
debug=False,
|
||||
device_id="emulator-5554",
|
||||
)
|
||||
|
||||
def mock_parse_args(self):
|
||||
self.args = args
|
||||
|
||||
monkeypatch.setattr(Config, "parse_args", mock_parse_args)
|
||||
|
||||
# Execute start_bot - it should naturally exit after 1 loop without infinite hanging!
|
||||
try:
|
||||
bot_flow.start_bot()
|
||||
finally:
|
||||
GoalExecutor.global_max_runtime_minutes = None
|
||||
GoalExecutor.global_start_time = None
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
DopamineEngine.global_max_runtime_minutes = None
|
||||
DopamineEngine.global_start_time = None
|
||||
|
||||
assert True
|
||||
|
||||
@@ -7,10 +7,11 @@ purged so it doesn't immediately trap itself again on the next iteration.
|
||||
"""
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
from tests.e2e.device_emulator import create_emulator_facade
|
||||
|
||||
def test_goap_recovers_from_trapped_state_with_restart(e2e_device, monkeypatch):
|
||||
|
||||
def test_goap_recovers_from_trapped_state_with_restart(monkeypatch):
|
||||
"""
|
||||
Simulates a broken UI where an action repeatedly fails.
|
||||
Verifies that GOAP triggers 'force start instagram', and successfully
|
||||
@@ -18,58 +19,38 @@ def test_goap_recovers_from_trapped_state_with_restart(e2e_device, monkeypatch):
|
||||
avoiding an infinite restart loop.
|
||||
"""
|
||||
xml = load_fixture_xml("home_feed_real.xml")
|
||||
|
||||
# We create a device that ALWAYS returns the same home_feed XML,
|
||||
# no matter what action is taken. This forces the UI to never change,
|
||||
# which causes actions to fail repeatedly.
|
||||
device = e2e_device([xml] * 20)
|
||||
|
||||
|
||||
# Single-state emulator with no transitions → UI never changes.
|
||||
# This forces actions to fail repeatedly, triggering GOAP trap detection.
|
||||
device, emulator = create_emulator_facade("trapped", {"trapped": xml}, {}, monkeypatch)
|
||||
|
||||
goap = GoalExecutor.get_instance(device, bot_username="testuser")
|
||||
goap.action_failures.clear()
|
||||
|
||||
# We intercept app_start to track how many times it was forced to restart
|
||||
restart_count = 0
|
||||
def mock_app_start(*args, **kwargs):
|
||||
nonlocal restart_count
|
||||
restart_count += 1
|
||||
|
||||
monkeypatch.setattr(device, "app_start", mock_app_start)
|
||||
|
||||
|
||||
# We want to ask it to reach REELS_FEED.
|
||||
# Since the UI never changes, 'tap reels tab' will fail.
|
||||
# It will fail twice, get masked, GOAP gets trapped, triggers restart.
|
||||
# We set max_steps to a small number so it doesn't loop forever in the test.
|
||||
# We just want to see that AFTER the restart, it tries 'tap reels tab' again,
|
||||
# meaning it cleared the state.
|
||||
|
||||
# Let's mock _execute_action slightly just to spy on it without changing behavior
|
||||
original_execute = goap._execute_action
|
||||
executed_actions = []
|
||||
|
||||
def spy_execute(action, goal=None):
|
||||
executed_actions.append(action)
|
||||
return original_execute(action, goal)
|
||||
|
||||
monkeypatch.setattr(goap, "_execute_action", spy_execute)
|
||||
|
||||
|
||||
goap.achieve("open reels", max_steps=6)
|
||||
|
||||
|
||||
# It should have tried 'tap reels tab' twice, failed both times,
|
||||
# then triggered 'force start instagram'.
|
||||
# Because of the fix, after the restart, it should have cleared failures
|
||||
# and tried 'tap reels tab' AGAIN.
|
||||
|
||||
assert restart_count > 0, "GOAP never attempted to force restart Instagram when trapped!"
|
||||
|
||||
# Count how many times it tried the action
|
||||
reels_attempts = executed_actions.count("tap reels tab")
|
||||
|
||||
|
||||
assert len(emulator.app_starts) > 0, "GOAP never attempted to force restart Instagram when trapped!"
|
||||
|
||||
# Count how many times it clicked the screen (trying to tap the reels tab)
|
||||
reels_attempts = len(emulator.clicks)
|
||||
|
||||
assert reels_attempts > 2, (
|
||||
f"GOAP got trapped, restarted, but never tried the action again! "
|
||||
f"It only tried {reels_attempts} times, meaning the memory leak is still there. "
|
||||
f"Actions executed: {executed_actions}"
|
||||
)
|
||||
|
||||
|
||||
# If the bug was present, it would only try 'tap reels tab' 2 times, mask it forever,
|
||||
# and then spam 'force start instagram' for the remaining steps.
|
||||
# With the fix, it tries 2 times, restarts, tries 2 times, restarts, etc.
|
||||
|
||||
@@ -10,19 +10,19 @@ from GramAddict.core.behaviors import BehaviorContext
|
||||
from tests.e2e.conftest import load_fixture_xml
|
||||
import pytest
|
||||
|
||||
class DummyDevice:
|
||||
def dump_hierarchy(self):
|
||||
return ""
|
||||
|
||||
def test_extract_username_from_home_feed():
|
||||
def test_extract_username_from_home_feed(
|
||||
make_real_device_with_xml, e2e_configs, e2e_session, e2e_cognitive_stack_factory
|
||||
):
|
||||
plugin = PostDataExtractionPlugin()
|
||||
xml = load_fixture_xml("home_feed_real.xml")
|
||||
|
||||
device = make_real_device_with_xml(xml)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=DummyDevice(),
|
||||
configs=None,
|
||||
session_state=None,
|
||||
cognitive_stack=None,
|
||||
device=device,
|
||||
configs=e2e_configs,
|
||||
session_state=e2e_session,
|
||||
cognitive_stack=e2e_cognitive_stack_factory(device),
|
||||
context_xml=xml,
|
||||
post_data={},
|
||||
username="",
|
||||
@@ -34,15 +34,19 @@ def test_extract_username_from_home_feed():
|
||||
assert ctx.username is not None
|
||||
assert ctx.post_data.get("username_missing") is not True
|
||||
|
||||
def test_extract_username_from_reels_feed():
|
||||
def test_extract_username_from_reels_feed(
|
||||
make_real_device_with_xml, e2e_configs, e2e_session, e2e_cognitive_stack_factory
|
||||
):
|
||||
plugin = PostDataExtractionPlugin()
|
||||
xml = load_fixture_xml("reels_feed_real.xml")
|
||||
|
||||
device = make_real_device_with_xml(xml)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=DummyDevice(),
|
||||
configs=None,
|
||||
session_state=None,
|
||||
cognitive_stack=None,
|
||||
device=device,
|
||||
configs=e2e_configs,
|
||||
session_state=e2e_session,
|
||||
cognitive_stack=e2e_cognitive_stack_factory(device),
|
||||
context_xml=xml,
|
||||
post_data={},
|
||||
username="",
|
||||
@@ -54,52 +58,4 @@ def test_extract_username_from_reels_feed():
|
||||
assert ctx.username is not None
|
||||
assert ctx.post_data.get("username_missing") is not True
|
||||
|
||||
def test_extract_username_with_vlm_fallback(monkeypatch):
|
||||
"""
|
||||
Tests that if the structural fast path fails, the VLM fallback correctly
|
||||
descends into ViewGroups if the VLM selects a container instead of the text node.
|
||||
"""
|
||||
plugin = PostDataExtractionPlugin()
|
||||
xml = load_fixture_xml("reels_feed_real.xml")
|
||||
|
||||
# Mock TelepathicEngine to return the container node (like in the logs)
|
||||
class FakeTelepath:
|
||||
def find_best_node(self, xml_str, prompt, **kwargs):
|
||||
if "author username" in prompt:
|
||||
# Return the clips_author_info_component (a ViewGroup with no text)
|
||||
return {
|
||||
"original_attribs": {
|
||||
"resource-id": "com.instagram.android:id/clips_author_info_component",
|
||||
"text": "",
|
||||
"content_desc": "",
|
||||
"bounds": "[42,1957][591,2098]"
|
||||
}
|
||||
}
|
||||
if "media content" in prompt:
|
||||
return {"original_attribs": {"content_desc": "Test media"}}
|
||||
return None
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
monkeypatch.setattr(TelepathicEngine, "get_instance", lambda *args, **kwargs: FakeTelepath())
|
||||
|
||||
# We must also mock the structural fast path so it falls through to VLM
|
||||
# by temporarily removing 'clips_author_username' from the fast path list during this test.
|
||||
# We will just mutate the xml to not have the structural IDs
|
||||
xml_no_fastpath = xml.replace("row_feed_photo_profile_name", "hidden_id")
|
||||
xml_no_fastpath = xml_no_fastpath.replace("clips_author_username", "hidden_id")
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=DummyDevice(),
|
||||
configs=None,
|
||||
session_state=None,
|
||||
cognitive_stack=None,
|
||||
context_xml=xml_no_fastpath,
|
||||
post_data={},
|
||||
username="",
|
||||
)
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
assert result.executed is True
|
||||
# The VLM selected the container. The engine should have parsed the children
|
||||
# and found "aditnugrahh" inside it.
|
||||
assert ctx.username == "aditnugrahh", f"Expected 'aditnugrahh' but got {ctx.username}"
|
||||
|
||||
|
||||
@@ -13,20 +13,15 @@ Design Philosophy:
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pkgutil
|
||||
import sys
|
||||
from argparse import Namespace
|
||||
from datetime import datetime, timedelta
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.session_state import SessionState, SessionStateEncoder
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# CONTRACT 1: SessionState Serialization is ALWAYS Safe
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -61,10 +56,9 @@ def _make_session_with_args(**extra_args):
|
||||
for k, v in extra_args.items():
|
||||
setattr(base_args, k, v)
|
||||
|
||||
class FakeConfig:
|
||||
pass
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
configs = FakeConfig()
|
||||
configs = Config(first_run=True)
|
||||
configs.args = base_args
|
||||
return SessionState(configs)
|
||||
|
||||
@@ -144,8 +138,6 @@ class TestPersistenceContract:
|
||||
"""
|
||||
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
|
||||
|
||||
from GramAddict.core.persistent_list import PersistentList
|
||||
|
||||
out_file = tmp_path / "test_sessions.json"
|
||||
|
||||
# Create a session with limits set (as bot_flow.py does)
|
||||
@@ -244,8 +236,8 @@ class TestProductionParityContract:
|
||||
unaudited.append(f"{fname}:{lineno} → {line.strip()}")
|
||||
|
||||
assert not unaudited, (
|
||||
f"UNAUDITED TEST DIVERGENCE DETECTED!\n"
|
||||
f"The following production code paths behave differently in test vs production:\n"
|
||||
"UNAUDITED TEST DIVERGENCE DETECTED!\n"
|
||||
"The following production code paths behave differently in test vs production:\n"
|
||||
+ "\n".join(f" ❌ {u}" for u in unaudited)
|
||||
+ "\n\nEach divergence is a potential 'lying test'. "
|
||||
"Add it to KNOWN_DIVERGENCES with a justification, or remove the guard."
|
||||
@@ -303,7 +295,6 @@ class TestImportIntegrityContract:
|
||||
except Exception as e:
|
||||
errors.append(f"{mod_path}: {type(e).__name__}: {e}")
|
||||
|
||||
assert not errors, (
|
||||
f"MODULE IMPORT FAILURES — The bot would crash on startup!\n"
|
||||
+ "\n".join(f" ❌ {e}" for e in errors)
|
||||
assert not errors, "MODULE IMPORT FAILURES — The bot would crash on startup!\n" + "\n".join(
|
||||
f" ❌ {e}" for e in errors
|
||||
)
|
||||
|
||||
@@ -28,36 +28,45 @@ from GramAddict.core.perception.action_memory import (
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Fake UIMemoryDB — replaces MagicMock to satisfy mock ban
|
||||
# Spy UIMemoryDB — wraps the REAL UIMemoryDB to track calls
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class FakeUIMemoryDB:
|
||||
class SpyUIMemoryDB:
|
||||
"""
|
||||
Real fake implementation of UIMemoryDB that tracks method calls
|
||||
without using unittest.mock. Satisfies the project's strict mock ban.
|
||||
Wraps the real UIMemoryDB to track method calls for assertions
|
||||
while exercising the actual production code path.
|
||||
|
||||
When Qdrant is down, UIMemoryDB gracefully degrades (client=None)
|
||||
and all operations become no-ops — which is the exact production
|
||||
behavior we want to test against.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
from GramAddict.core.qdrant_memory import UIMemoryDB
|
||||
|
||||
self._real = UIMemoryDB()
|
||||
self.store_memory_calls = []
|
||||
self.boost_confidence_calls = []
|
||||
self.decay_confidence_calls = []
|
||||
self.retrieve_memory_calls = []
|
||||
|
||||
def retrieve_memory(self, intent, xml_context):
|
||||
def retrieve_memory(self, intent, xml_context, **kwargs):
|
||||
self.retrieve_memory_calls.append((intent, xml_context))
|
||||
return None
|
||||
return self._real.retrieve_memory(intent, xml_context, **kwargs)
|
||||
|
||||
def store_memory(self, intent, xml_context, node_dict):
|
||||
self.store_memory_calls.append((intent, xml_context, node_dict))
|
||||
self._real.store_memory(intent, xml_context, node_dict)
|
||||
|
||||
def boost_confidence(self, intent, xml_context):
|
||||
def boost_confidence(self, intent, xml_context=None, **kwargs):
|
||||
self.boost_confidence_calls.append((intent, xml_context))
|
||||
self._real.boost_confidence(intent, xml_context, **kwargs)
|
||||
|
||||
def decay_confidence(self, intent, xml_context):
|
||||
def decay_confidence(self, intent, xml_context=None, **kwargs):
|
||||
self.decay_confidence_calls.append((intent, xml_context))
|
||||
self._real.decay_confidence(intent, xml_context, **kwargs)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -166,9 +175,7 @@ class TestVLMResponseParsing:
|
||||
)
|
||||
def test_parse_yes_no(self, response, expected):
|
||||
result = _parse_yes_no(response)
|
||||
assert result is expected, (
|
||||
f"_parse_yes_no('{response}') returned {result}, expected {expected}"
|
||||
)
|
||||
assert result is expected, f"_parse_yes_no('{response}') returned {result}, expected {expected}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -266,9 +273,7 @@ class TestStructuralGuards:
|
||||
bounds=(216, 2300, 432, 2400),
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "tap explore tab", screen_height=2400
|
||||
)
|
||||
filtered = self.resolver.filter_navigation_conflicts(candidates, "tap explore tab", screen_height=2400)
|
||||
assert len(filtered) == 1
|
||||
assert filtered[0].center_y == 2350
|
||||
|
||||
@@ -289,9 +294,7 @@ class TestStructuralGuards:
|
||||
center_y=2350,
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "tap post author username", screen_height=2400
|
||||
)
|
||||
filtered = self.resolver.filter_navigation_conflicts(candidates, "tap post author username", screen_height=2400)
|
||||
author_nodes = [n for n in filtered if n.text == "photographer_jane"]
|
||||
assert len(author_nodes) == 1
|
||||
|
||||
@@ -323,7 +326,8 @@ class TestStructuralGuards:
|
||||
),
|
||||
]
|
||||
filtered = self.resolver.filter_navigation_conflicts(
|
||||
candidates, "post author username text (exclude bottom tabs)",
|
||||
candidates,
|
||||
"post author username text (exclude bottom tabs)",
|
||||
screen_height=2400,
|
||||
)
|
||||
# The author username at y=2012 MUST survive
|
||||
@@ -462,8 +466,7 @@ class TestSemanticMatchGuard:
|
||||
def test_intent_matches_node(self, intent, semantic_string, expected):
|
||||
result = _intent_matches_node(intent, semantic_string)
|
||||
assert result is expected, (
|
||||
f"_intent_matches_node('{intent}', '{semantic_string[:50]}...') "
|
||||
f"returned {result}, expected {expected}"
|
||||
f"_intent_matches_node('{intent}', '{semantic_string[:50]}...') " f"returned {result}, expected {expected}"
|
||||
)
|
||||
|
||||
|
||||
@@ -482,9 +485,9 @@ class TestActionMemoryLifecycle:
|
||||
"""Tests the full click tracking → confirmation/rejection lifecycle."""
|
||||
|
||||
def _make_memory(self):
|
||||
"""Create ActionMemory with a FakeUIMemoryDB."""
|
||||
fake_db = FakeUIMemoryDB()
|
||||
return ActionMemory(ui_memory=fake_db), fake_db
|
||||
"""Create ActionMemory with a SpyUIMemoryDB wrapping the real UIMemoryDB."""
|
||||
spy_db = SpyUIMemoryDB()
|
||||
return ActionMemory(ui_memory=spy_db), spy_db
|
||||
|
||||
def test_confirm_correct_follow_stores_in_memory(self):
|
||||
"""A confirmed 'follow' click on a Follow button must be stored."""
|
||||
@@ -539,9 +542,7 @@ class TestActionMemoryLifecycle:
|
||||
memory, _ = self._make_memory()
|
||||
|
||||
post_xml_success = '<node text="Following" />'
|
||||
result = memory.verify_success(
|
||||
"follow", pre_click_xml="<node/>", post_click_xml=post_xml_success
|
||||
)
|
||||
result = memory.verify_success("follow", pre_click_xml="<node/>", post_click_xml=post_xml_success)
|
||||
assert result is True
|
||||
|
||||
def test_verify_view_post_success(self):
|
||||
@@ -549,9 +550,7 @@ class TestActionMemoryLifecycle:
|
||||
memory, _ = self._make_memory()
|
||||
|
||||
post_xml_success = '<node resource-id="com.instagram.android:id/row_feed_button_like" />'
|
||||
result = memory.verify_success(
|
||||
"view a post", pre_click_xml="<node/>", post_click_xml=post_xml_success
|
||||
)
|
||||
result = memory.verify_success("view a post", pre_click_xml="<node/>", post_click_xml=post_xml_success)
|
||||
assert result is True
|
||||
|
||||
|
||||
@@ -828,8 +827,8 @@ class TestVerifySuccessStructuralDelta:
|
||||
"""Tests the structural XML diff logic in verify_success."""
|
||||
|
||||
def _make_memory(self):
|
||||
fake_db = FakeUIMemoryDB()
|
||||
return ActionMemory(ui_memory=fake_db), fake_db
|
||||
spy_db = SpyUIMemoryDB()
|
||||
return ActionMemory(ui_memory=spy_db), spy_db
|
||||
|
||||
def test_toggle_massive_shift_is_navigation_error(self):
|
||||
"""
|
||||
@@ -917,4 +916,3 @@ class TestVerifySuccessStructuralDelta:
|
||||
|
||||
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
|
||||
assert result is False
|
||||
|
||||
|
||||
@@ -9,26 +9,18 @@ without relying on brittle mock LLM responses.
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
from tests.e2e.device_emulator import create_emulator_facade
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sae(monkeypatch):
|
||||
def sae(make_real_device_with_xml):
|
||||
SituationalAwarenessEngine.reset()
|
||||
device, emulator = create_emulator_facade("any", {"any": ""}, {}, monkeypatch)
|
||||
device = make_real_device_with_xml("")
|
||||
engine = SituationalAwarenessEngine.get_instance(device)
|
||||
# Clear the global ScreenMemoryDB so tests don't pollute each other
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
db = ScreenMemoryDB()
|
||||
if db.is_connected:
|
||||
try:
|
||||
db.client.delete(
|
||||
collection_name=db.collection_name,
|
||||
points_selector={"filter": {}}, # Delete all points
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
db.wipe_collection()
|
||||
return engine
|
||||
|
||||
|
||||
@@ -40,7 +32,7 @@ class TestSAEPerception:
|
||||
assert sae.perceive("") == SituationType.OBSTACLE_FOREIGN_APP
|
||||
assert sae.perceive(None) == SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
def test_perceive_locked_screen(self, sae, monkeypatch):
|
||||
def test_perceive_locked_screen(self, sae):
|
||||
"""If the hardware reports the screen is off, it must be LOCKED_SCREEN."""
|
||||
sae.device.deviceV2.info = {"screenOn": False}
|
||||
xml_dump = '<node package="com.android.systemui" />'
|
||||
@@ -134,7 +126,7 @@ class TestSAELoop:
|
||||
"""
|
||||
# Inject the XML into the emulator's state instead of bypassing dump_hierarchy
|
||||
sae.device.deviceV2.info["screenOn"] = True
|
||||
sae.device.deviceV2._parent.states[sae.device.deviceV2._parent.current_state] = xml_dump
|
||||
sae.device.deviceV2.xml = xml_dump
|
||||
assert sae.ensure_clear_screen(max_attempts=3) is True
|
||||
assert sae._consecutive_failures == 0
|
||||
|
||||
@@ -148,3 +140,44 @@ class TestSAELoop:
|
||||
|
||||
foreign_xml = '<node package="com.apple.ios" />'
|
||||
assert sae.is_instagram_foreground(xml_dump=foreign_xml) is False
|
||||
|
||||
def test_false_positive_modal_infinite_loop_trap(self, sae, monkeypatch):
|
||||
"""
|
||||
Reproduces a production trap where ScreenIdentity overrides Qdrant for structural
|
||||
markers (Priority 0). If the LLM identifies the modal as a false positive, it unlearns
|
||||
it in Qdrant. However, without the fix, the next time GOAP evaluates the state,
|
||||
ScreenIdentity STILL says MODAL because of Priority 0, triggering an infinite loop.
|
||||
This test ensures ScreenIdentity respects NORMAL memory overrides.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
# Load real home feed XML
|
||||
xml_dump = Path("tests/e2e/fixtures/home_feed_real.xml").read_text()
|
||||
|
||||
# Inject the Priority 0 structural marker
|
||||
xml_dump = xml_dump.replace(
|
||||
'<node index="0" text="" resource-id="android:id/content"',
|
||||
'<node resource-id="com.instagram.android:id/gallery_cancel_button" bounds="[0,0][100,100]" />\n<node index="0" text="" resource-id="android:id/content"',
|
||||
)
|
||||
|
||||
sae.device.deviceV2.xml = xml_dump
|
||||
sae.device.deviceV2.info["screenOn"] = True
|
||||
|
||||
# Simulate LLM unlearning by storing this exact state as NORMAL
|
||||
compressed = sae._compress_xml(xml_dump)
|
||||
ScreenMemoryDB().store_screen(compressed, "NORMAL")
|
||||
|
||||
identity = ScreenIdentity("testuser")
|
||||
# Ensure we inject the device so get_screenshot_b64 doesn't crash if it falls back
|
||||
identity.device = sae.device
|
||||
|
||||
# The bug: this would return MODAL because of Priority 0, ignoring the DB
|
||||
# The fix: it should return HOME_FEED because is_normal_override = True skips the MODAL check
|
||||
result = identity.identify(xml_dump)
|
||||
|
||||
assert (
|
||||
result["screen_type"].value == "home_feed"
|
||||
), f"Infinite loop trap: Expected home_feed, got {result['screen_type'].value}"
|
||||
|
||||
@@ -70,3 +70,31 @@ def test_permission_dialog_terminates_chain(make_real_device_with_xml, e2e_workf
|
||||
|
||||
# BACK must have been pressed
|
||||
assert "back" in device.pressed_keys, "obstacle_guard did not press BACK — the dialog stays on screen!"
|
||||
|
||||
|
||||
def test_sae_escapes_permission_dialog_via_vlm(make_real_device_with_xml):
|
||||
"""
|
||||
Ensures that the SituationalAwarenessEngine's ensure_clear_screen loop
|
||||
can correctly use the VLM to escape an OBSTACLE_SYSTEM dialog.
|
||||
"""
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
# First XML is the permission dialog, second is the normal feed.
|
||||
device = make_real_device_with_xml([PERMISSION_DIALOG_XML, NORMAL_POST_XML])
|
||||
sae = SituationalAwarenessEngine.get_instance(device)
|
||||
|
||||
# We must wipe Qdrant memory for this situation so it forces an LLM call.
|
||||
_ = sae._compress_xml(PERMISSION_DIALOG_XML)
|
||||
sae.episodes.recall = lambda x: None # Force LLM instead of recalled memory for test determinism
|
||||
|
||||
# Ensure clear screen MUST return True, meaning it successfully cleared the obstacle.
|
||||
success = sae.ensure_clear_screen(max_attempts=3, initial_xml=PERMISSION_DIALOG_XML)
|
||||
|
||||
assert success is True, "ensure_clear_screen failed to escape the system dialog!"
|
||||
|
||||
# Verify that the LLM decided to either click the 'Don't allow' button or press BACK.
|
||||
# The 'Don't allow' button is at [100,1240][980,1340], center is (540, 1290).
|
||||
clicked_deny = any(abs(x - 540) < 50 and abs(y - 1290) < 50 for x, y in device.clicks)
|
||||
pressed_back = "back" in device.pressed_keys
|
||||
|
||||
assert clicked_deny or pressed_back, "VLM failed to click the 'Don't allow' button or press back!"
|
||||
|
||||
Reference in New Issue
Block a user