Compare commits
25 Commits
f220e09193
...
fix/purge-
| Author | SHA1 | Date | |
|---|---|---|---|
| 6abb519e3b | |||
| 4e91db01c9 | |||
| e55abc5a8a | |||
| 03105437b8 | |||
| b9c29a5a2d | |||
| 71310b8e84 | |||
| 073a90c38c | |||
| 44fae37cc7 | |||
| 48071cc9b8 | |||
| 10a85a91f1 | |||
| 5bf0053884 | |||
| a846462d02 | |||
| 0dbafd0a82 | |||
| 83e5b94ddf | |||
| dd8285e1ce | |||
| ac5d5351a6 | |||
| ad012b4cd4 | |||
| 5fcf1f180b | |||
| 9a74d89477 | |||
| dc4b576bc1 | |||
| e94dfe8c5c | |||
| 7aa6bfccf6 | |||
| 5fef014cb4 | |||
| 0bdfd999d2 | |||
| 4ad559e107 |
@@ -21,6 +21,7 @@ from GramAddict.core.dojo_engine import DojoEngine
|
||||
|
||||
# Cognitive Stack
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.log import configure_logger
|
||||
from GramAddict.core.perception.feed_analysis import (
|
||||
@@ -188,7 +189,6 @@ def start_bot(**kwargs):
|
||||
active_inference = ActiveInferenceEngine(username)
|
||||
|
||||
# Core Autonomous Engines
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
GoalExecutor.get_instance(device, username)
|
||||
zero_engine = ZeroLatencyEngine(device)
|
||||
@@ -349,9 +349,7 @@ def start_bot(**kwargs):
|
||||
logger.info(
|
||||
f"🧠 [Agent Orchestrator] Session started. Strategy: {growth_brain.strategy} | Persona: {getattr(configs.args, 'agent_persona', 'unknown')}"
|
||||
)
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
# 1. Starten wir den GOAP Executor, um die UI-Struktur autonom zu erfassen
|
||||
goap = GoalExecutor.get_instance(device, username)
|
||||
|
||||
# --- PHASE 0: Autonomous Profile Scanning ---
|
||||
@@ -447,10 +445,13 @@ def start_bot(**kwargs):
|
||||
has_scanned_own_profile = True
|
||||
|
||||
while not dopamine.is_app_session_over():
|
||||
# 1. Ask the Growth Brain for a Desire
|
||||
current_desire = growth_brain.get_current_desire(dopamine)
|
||||
# 1. Ask the Growth Brain for a Strategic Objective
|
||||
success_rates = getattr(session_state, "successfulInteractions", {})
|
||||
current_goal = growth_brain.get_current_goal(
|
||||
dopamine, getattr(configs.args, "goals", []), success_rates=success_rates
|
||||
)
|
||||
|
||||
if current_desire == "ShiftContext":
|
||||
if current_goal == "ShiftContext":
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart to clear context.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
@@ -459,6 +460,24 @@ def start_bot(**kwargs):
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
|
||||
# 2. Execution: GOAP Plan & Execute (Autonomous Mode)
|
||||
if getattr(configs.args, "goals", None):
|
||||
logger.info(f"🤖 Autonomous Mode Active. Delegating to GoalExecutor for: {current_goal}")
|
||||
|
||||
goal_executor = GoalExecutor(device=device, bot_username=getattr(configs.args, "username", ""))
|
||||
|
||||
result = goal_executor.achieve(current_goal)
|
||||
|
||||
if result:
|
||||
logger.info("✅ Goal achieved autonomously!")
|
||||
else:
|
||||
logger.warning(f"⚠️ Goal execution failed for: {current_goal}")
|
||||
|
||||
continue # The GoalExecutor handles navigation internally
|
||||
|
||||
# --- LEGACY PROCEDURAL FALLBACK (For config without goals) ---
|
||||
current_desire = current_goal
|
||||
|
||||
# 2. Map Desire to Sub-Feed
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
|
||||
@@ -85,6 +85,9 @@ class Config:
|
||||
self.username = self.username[0]
|
||||
self.debug = self.config.get("debug", False)
|
||||
self.app_id = self.config.get("app_id", "com.instagram.android")
|
||||
|
||||
# Autonomous Agent Goals
|
||||
self.goals = self.config.get("goals", [])
|
||||
else:
|
||||
if "--debug" in self.args:
|
||||
self.debug = True
|
||||
|
||||
@@ -13,20 +13,20 @@ MAX_REPLIES_PER_INBOX_VISIT = 3
|
||||
# Sentinel values that indicate missing message context.
|
||||
_EMPTY_CONTEXT_SENTINELS = frozenset({"no previous context", "", "none", "n/a"})
|
||||
|
||||
|
||||
# Structural resource-IDs that indicate a real "Send" button.
|
||||
_SEND_BUTTON_MARKERS = frozenset({"send_button", "row_thread_composer_send"})
|
||||
|
||||
|
||||
def _is_send_button(node: dict) -> bool:
|
||||
"""Structural verification: returns True only if the node is a real Send button."""
|
||||
attribs = node.get("original_attribs", {})
|
||||
rid = attribs.get("resource-id", "")
|
||||
desc = attribs.get("content-desc", node.get("desc", "")).lower()
|
||||
# Accept if resource-id contains a known send button marker
|
||||
if any(marker in rid for marker in _SEND_BUTTON_MARKERS):
|
||||
"""Semantic verification: returns True if the node is identified as a Send button."""
|
||||
desc = (node.get("description") or node.get("desc", "")).lower()
|
||||
text = (node.get("text") or "").lower()
|
||||
rid = (node.get("id") or node.get("resource_id", "")).lower()
|
||||
|
||||
# Accept if semantic markers indicate sending
|
||||
if any(m in rid for m in ["send", "composer_button"]):
|
||||
return True
|
||||
# Accept if content-desc is exactly "Send" (Instagram's canonical label)
|
||||
if desc == "send":
|
||||
if any(m in desc for m in ["send", "absenden"]):
|
||||
return True
|
||||
if text == "send" or text == "absenden":
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -83,16 +83,14 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
# --- Zero Trust Structural Guard ---
|
||||
# -----------------------------------
|
||||
# ZERO TRUST STRUCTURAL GUARD
|
||||
# -----------------------------------
|
||||
# Validate we are actually in the Inbox or a Thread.
|
||||
# Hallucinations can lead to "Privacy Settings" or "Profile" screens.
|
||||
is_inbox = (
|
||||
'resource-id="com.instagram.android:id/inbox_refreshable_thread_list_recyclerview"' in xml_dump
|
||||
or 'resource-id="com.instagram.android:id/direct_inbox_action_bar"' in xml_dump
|
||||
)
|
||||
is_thread = 'resource-id="com.instagram.android:id/direct_thread_header"' in xml_dump
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
identity_engine = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
screen_info = identity_engine.identify(xml_dump)
|
||||
|
||||
screen_type = screen_info["screen_type"]
|
||||
is_inbox = screen_type == ScreenType.DM_INBOX
|
||||
is_thread = screen_type == ScreenType.DM_THREAD
|
||||
|
||||
if is_thread:
|
||||
logger.warning("⚠️ [Structural Guard] DM Engine trapped in an open thread. Escaping...")
|
||||
@@ -102,9 +100,11 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
sleep(1.5)
|
||||
continue
|
||||
|
||||
if not is_inbox and not is_thread:
|
||||
if not is_inbox:
|
||||
# We have drifted somewhere entirely alien (like Privacy Settings)
|
||||
logger.error("🛑 [Structural Guard] Alien context detected. Not in Inbox. Triggering CONTEXT_LOST.")
|
||||
logger.error(
|
||||
f"🛑 [Structural Guard] Alien context detected ({screen_type}). Not in Inbox. Triggering CONTEXT_LOST."
|
||||
)
|
||||
return "CONTEXT_LOST"
|
||||
# -----------------------------------
|
||||
|
||||
@@ -215,10 +215,12 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
|
||||
# If keyboard was open, the first back only closed it. Check if still in thread.
|
||||
check_xml = device.dump_hierarchy()
|
||||
if (
|
||||
'resource-id="com.instagram.android:id/direct_thread_header"' in check_xml
|
||||
or 'resource-id="com.instagram.android:id/row_thread_composer_edittext"' in check_xml
|
||||
):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
@@ -239,10 +241,12 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
sleep(1.0)
|
||||
|
||||
check_xml = device.dump_hierarchy()
|
||||
if (
|
||||
'resource-id="com.instagram.android:id/direct_thread_header"' in check_xml
|
||||
or 'resource-id="com.instagram.android:id/row_thread_composer_edittext"' in check_xml
|
||||
):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
|
||||
@@ -17,11 +17,11 @@ import logging
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from GramAddict.core.utils import random_sleep
|
||||
from GramAddict.core.navigation.knowledge import NavigationKnowledge
|
||||
from GramAddict.core.navigation.path_memory import PathMemory
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
from GramAddict.core.utils import random_sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -117,10 +117,12 @@ class GoalExecutor:
|
||||
consecutive_back_presses = 0
|
||||
MAX_CONSECUTIVE_BACK = 3
|
||||
explored_nav_actions = set()
|
||||
visited_screens = set()
|
||||
for step_num in range(max_steps):
|
||||
# PERCEIVE
|
||||
screen = self.perceive()
|
||||
screen_type = screen["screen_type"]
|
||||
visited_screens.add(screen_type)
|
||||
|
||||
if last_screen_type and screen_type != last_screen_type:
|
||||
logger.debug(
|
||||
@@ -144,7 +146,7 @@ class GoalExecutor:
|
||||
screen["available_actions"] = masked_available
|
||||
|
||||
logger.debug(
|
||||
f"📍 [GOAP Step {step_num + 1}] On: {screen_type.value} | "
|
||||
f"📍 [GOAP Step {step_num + 1}] Goal: '{goal}' | On: {screen_type.value} | "
|
||||
f"Available: {screen.get('available_actions', [])[:5]}"
|
||||
)
|
||||
|
||||
@@ -172,7 +174,11 @@ class GoalExecutor:
|
||||
|
||||
# PLAN
|
||||
action = self.planner.plan_next_step(
|
||||
goal, screen, explored_nav_actions=explored_nav_actions, action_failures=self.action_failures
|
||||
goal,
|
||||
screen,
|
||||
explored_nav_actions=explored_nav_actions,
|
||||
action_failures=self.action_failures,
|
||||
visited_screens=visited_screens,
|
||||
)
|
||||
|
||||
if action is None:
|
||||
@@ -356,9 +362,16 @@ class GoalExecutor:
|
||||
# Determine if this was a navigation or an interaction
|
||||
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
|
||||
action_success = False
|
||||
ui_changed = post_xml != xml_dump
|
||||
|
||||
# ── UI Change Detection with Noise Threshold ──
|
||||
# Raw string diffs of < 50 bytes are noise (timestamps, whitespace, counters).
|
||||
# A real navigation changes the XML by hundreds/thousands of bytes.
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
xml_delta = abs(len(post_xml) - len(xml_dump))
|
||||
ui_changed = post_xml != xml_dump and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
logger.debug(
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, " f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}"
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, "
|
||||
f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}, delta={xml_delta}b"
|
||||
)
|
||||
|
||||
if is_navigation:
|
||||
|
||||
@@ -94,6 +94,33 @@ class GrowthBrain:
|
||||
logger.info(f"🧠 [GrowthBrain] Strategy '{self.strategy}' dictated Desire: {selected_desire}")
|
||||
return selected_desire
|
||||
|
||||
def get_current_goal(self, dopamine_engine, available_goals: list[str], success_rates: dict = None) -> str:
|
||||
"""
|
||||
Autonomously selects the next strategic goal.
|
||||
If no goals are configured, falls back to legacy desires.
|
||||
Weights goals based on session success rates if provided.
|
||||
"""
|
||||
import random
|
||||
|
||||
if not available_goals:
|
||||
# Legacy Desire Mapping (Fallback)
|
||||
return self.get_current_desire(dopamine_engine)
|
||||
|
||||
if dopamine_engine.boredom > 80:
|
||||
return "ShiftContext" # High boredom triggers a context shift
|
||||
|
||||
if not success_rates:
|
||||
return random.choice(available_goals)
|
||||
|
||||
weights = []
|
||||
for goal in available_goals:
|
||||
base_weight = 1.0
|
||||
success_count = success_rates.get(goal, 0)
|
||||
weight = base_weight + float(success_count)
|
||||
weights.append(weight)
|
||||
|
||||
return random.choices(available_goals, weights=weights, k=1)[0]
|
||||
|
||||
def get_circadian_pacing(self) -> float:
|
||||
"""
|
||||
Adjusts activity levels based on the current local time
|
||||
|
||||
@@ -287,6 +287,12 @@ def query_llm(
|
||||
req_data["images"] = images_b64
|
||||
if format_json:
|
||||
req_data["format"] = "json"
|
||||
else:
|
||||
# For free-text calls (Brain action extraction), explicitly disable
|
||||
# thinking mode. Reasoning models like qwen3.5 put EVERYTHING in
|
||||
# the thinking block and return response='', which is useless for
|
||||
# action extraction. think=false forces a direct response.
|
||||
req_data["think"] = False
|
||||
|
||||
# Ollama passes configs inside 'options'
|
||||
if temperature is not None or max_tokens is not None:
|
||||
@@ -344,16 +350,27 @@ def query_llm(
|
||||
return {"response": content}
|
||||
else:
|
||||
# Ollama returns response OR thinking (for reasoning models)
|
||||
content = resp_json.get("response") or resp_json.get("thinking") or ""
|
||||
raw_response = resp_json.get("response", "")
|
||||
raw_thinking = resp_json.get("thinking", "")
|
||||
|
||||
logger.debug(f"DEBUG LLM PAYLOAD: response='{raw_response}', thinking='{raw_thinking}'")
|
||||
|
||||
# CRITICAL: For free-text mode (format_json=False), do NOT substitute
|
||||
# thinking for empty response. The thinking block is REASONING, not
|
||||
# a decision. The Brain parser would extract random actions from it.
|
||||
# For JSON mode (format_json=True), falling back to thinking IS correct
|
||||
# because reasoning models may place structured output in the thinking block.
|
||||
if format_json:
|
||||
content = raw_response or raw_thinking or ""
|
||||
extracted = extract_json(content)
|
||||
if not extracted:
|
||||
# Log more context if JSON extraction fails
|
||||
logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...")
|
||||
raise ValueError("Ollama returned non-JSON content when JSON was expected.")
|
||||
resp_json["response"] = extracted
|
||||
logger.warning(f"Failed to extract JSON from content: {content[:100]}")
|
||||
else:
|
||||
content = extracted
|
||||
else:
|
||||
content = raw_response
|
||||
|
||||
return resp_json
|
||||
return {"response": content}
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?")
|
||||
except Exception as e:
|
||||
|
||||
@@ -15,7 +15,11 @@ def ask_brain_for_action(
|
||||
return None
|
||||
|
||||
cfg = Config()
|
||||
url = getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate") if hasattr(cfg, "args") else "http://localhost:11434/api/generate"
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
prompt = (
|
||||
@@ -32,7 +36,7 @@ def ask_brain_for_action(
|
||||
"INSTRUCTIONS:\n"
|
||||
"1. Reason about where you are. Consider the screen type and what actions make sense on that screen.\n"
|
||||
"2. If the goal requires navigating away from the current screen, choose the action that moves you closest to the goal.\n"
|
||||
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). Some screens like stories or modals are NOT scrollable.\n"
|
||||
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). If your target is likely on this screen but not currently visible, you MUST choose 'scroll down'.\n"
|
||||
"4. 'press back' exits the current screen and returns to the previous one. Use it when you are on a screen that doesn't lead to your goal.\n"
|
||||
"5. DO NOT hallucinate actions. Reply ONLY with the exact string from the available actions list.\n"
|
||||
"6. Reply with ONLY the action string, nothing else."
|
||||
@@ -40,18 +44,43 @@ def ask_brain_for_action(
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False
|
||||
url=url,
|
||||
model=model,
|
||||
prompt="Choose the next best action.",
|
||||
system=prompt,
|
||||
format_json=False,
|
||||
max_tokens=250,
|
||||
)
|
||||
if response:
|
||||
result = response if isinstance(response, str) else response.get("response", "")
|
||||
result = result.strip().strip("'\"")
|
||||
|
||||
# Fuzzy match to available actions just in case
|
||||
# 1. Exact match check (ideal case)
|
||||
for act in available_actions:
|
||||
if act.lower() in result.lower():
|
||||
if act.lower() == result.lower():
|
||||
return act
|
||||
|
||||
# 2. Strict line-by-line check (often the model outputs the action on the last line)
|
||||
for line in reversed(result.splitlines()):
|
||||
line = line.strip().strip("'\"")
|
||||
for act in available_actions:
|
||||
if act.lower() == line.lower():
|
||||
return act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action: '{result}'. Falling back.")
|
||||
# 3. Fuzzy match (find the LAST mentioned action in the text, assuming it's the conclusion)
|
||||
best_act = None
|
||||
best_idx = -1
|
||||
for act in available_actions:
|
||||
idx = result.lower().rfind(act.lower())
|
||||
if idx > best_idx:
|
||||
best_idx = idx
|
||||
best_act = act
|
||||
|
||||
if best_act:
|
||||
logger.warning(f"🧠 [Brain] Extracted action '{best_act}' from verbose LLM output.")
|
||||
return best_act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back.")
|
||||
except Exception as e:
|
||||
logger.debug(f"🧠 [Brain] Error querying LLM: {e}")
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ class PathMemory:
|
||||
try:
|
||||
from qdrant_client import models
|
||||
|
||||
point_id = self._db._get_id(seed)
|
||||
point_id = self._db.generate_uuid(seed)
|
||||
self._db.client.delete(
|
||||
collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id])
|
||||
)
|
||||
|
||||
@@ -18,7 +18,12 @@ class GoalPlanner:
|
||||
self.knowledge = NavigationKnowledge(username)
|
||||
|
||||
def plan_next_step(
|
||||
self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None, action_failures: dict = None
|
||||
self,
|
||||
goal: str,
|
||||
screen: Dict[str, Any],
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""Plans the NEXT single action to take toward the goal."""
|
||||
screen_type = screen["screen_type"]
|
||||
@@ -37,7 +42,7 @@ class GoalPlanner:
|
||||
# ── 3. Am I on the right screen? If not, navigate there ──
|
||||
selected_tab = screen.get("selected_tab")
|
||||
nav_action = self._plan_navigation(
|
||||
goal_lower, screen_type, available, selected_tab, explored_nav_actions, action_failures
|
||||
goal_lower, screen_type, available, selected_tab, explored_nav_actions, action_failures, visited_screens
|
||||
)
|
||||
if nav_action:
|
||||
return nav_action
|
||||
@@ -75,6 +80,7 @@ class GoalPlanner:
|
||||
selected_tab: Optional[str] = None,
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""If we're on the wrong screen, figure out how to navigate.
|
||||
|
||||
@@ -94,6 +100,37 @@ class GoalPlanner:
|
||||
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
|
||||
available = safe_available
|
||||
|
||||
visited_screens = visited_screens or set()
|
||||
|
||||
# 0b. No-Op Guard & Anti-Loop Guard:
|
||||
# - Strip tab actions that navigate to the CURRENT screen.
|
||||
# - Strip actions that navigate to PREVIOUSLY VISITED screens (except back-tracking).
|
||||
noop_actions = set()
|
||||
for action in available:
|
||||
expected = ScreenTopology.expected_screen_for_action(action, screen_type)
|
||||
if expected == screen_type:
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [No-Op Guard] Stripping '{action}' — leads back to {screen_type.name}")
|
||||
elif expected in visited_screens and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — leads to visited {expected.name}")
|
||||
|
||||
# Also strip actions where the HD Map says they go TO the current screen from OTHER screens
|
||||
for src_screen, transitions in ScreenTopology.TRANSITIONS.items():
|
||||
if src_screen == screen_type:
|
||||
continue # We already handled this screen's own transitions
|
||||
for action, dest in transitions.items():
|
||||
if dest == screen_type and action in available:
|
||||
noop_actions.add(action)
|
||||
logger.debug(
|
||||
f"🛡️ [No-Op Guard] Stripping '{action}' — known to navigate to current {screen_type.name}"
|
||||
)
|
||||
elif dest in visited_screens and action in available and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — known to navigate to visited {dest.name}")
|
||||
|
||||
available = [a for a in available if a not in noop_actions]
|
||||
|
||||
# Build avoid_actions for HD Map route planning
|
||||
avoid_actions = (explored_nav_actions or set()).copy()
|
||||
if action_failures:
|
||||
@@ -102,14 +139,16 @@ class GoalPlanner:
|
||||
avoid_actions.add(act)
|
||||
|
||||
target_screen = ScreenTopology.goal_to_target_screen(goal)
|
||||
|
||||
|
||||
# ── 1. HD Map Pre-Check for Dead Ends ──
|
||||
# If the topological map KNOWS the target is unreachable due to action_failures,
|
||||
# we must preempt the Brain from blindly routing into a dead end.
|
||||
if target_screen and target_screen != screen_type:
|
||||
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
|
||||
if route is None and ScreenTopology.find_route(screen_type, target_screen):
|
||||
logger.warning(f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing.")
|
||||
logger.warning(
|
||||
f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── 2. Brain-Driven Decision Making (Primary Strategy) ──
|
||||
@@ -118,7 +157,7 @@ class GoalPlanner:
|
||||
|
||||
brain_action = ask_brain_for_action(goal, screen_type.name, available, avoid_actions)
|
||||
if brain_action:
|
||||
logger.info(f"🧠 [Brain] Decided dynamically to execute: '{brain_action}'")
|
||||
logger.info(f"🧠 [Brain] Decided to execute: '{brain_action}' (to achieve: '{goal}')")
|
||||
return brain_action
|
||||
|
||||
# ── 2. HD Map Routing (Fallback) ──
|
||||
|
||||
@@ -71,7 +71,10 @@ class ActionMemory:
|
||||
self._last_click_context = None
|
||||
return
|
||||
|
||||
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
|
||||
logger.info(
|
||||
f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.",
|
||||
extra={"color": "\x1b[32m"}
|
||||
)
|
||||
|
||||
# Store or boost in Qdrant
|
||||
try:
|
||||
@@ -95,7 +98,10 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
logger.warning(f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.")
|
||||
logger.warning(
|
||||
f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.",
|
||||
extra={"color": "\x1b[31m"}
|
||||
)
|
||||
|
||||
try:
|
||||
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
|
||||
@@ -113,11 +119,11 @@ class ActionMemory:
|
||||
intent_lower = intent.lower()
|
||||
post_xml_lower = post_click_xml.lower()
|
||||
|
||||
# Specific check for explore grid
|
||||
if "first image in explore grid" in intent_lower or "grid item" in intent_lower:
|
||||
if "row_feed_photo_imageview" in post_xml_lower or "row_feed_button_like" in post_xml_lower:
|
||||
# Specific check for opening a post (from explore/profile grid)
|
||||
if "view a post" in intent_lower or "first image" in intent_lower or "grid item" in intent_lower:
|
||||
if "row_feed_photo_imageview" in post_xml_lower or "row_feed_button_like" in post_xml_lower or "clips_viewer_view_pager" in post_xml_lower:
|
||||
return True
|
||||
if "explore_action_bar" in post_xml_lower and "row_feed_button_like" not in post_xml_lower:
|
||||
if "explore_action_bar" in post_xml_lower and "row_feed_button_like" not in post_xml_lower and "clips_viewer" not in post_xml_lower:
|
||||
return None # Still on grid, inconclusive
|
||||
|
||||
state_toggles = ["like", "save", "follow", "heart"]
|
||||
@@ -208,14 +214,43 @@ class ActionMemory:
|
||||
logger.debug(f"🧠 [ActionMemory] Structural delta detected for toggle '{intent}'. Verification PASS.")
|
||||
return True
|
||||
else:
|
||||
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
|
||||
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
|
||||
# also produces a large diff but achieves the wrong goal.
|
||||
if diff > 50:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
# Is it a standard structural transition?
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
|
||||
return False
|
||||
# We don't have screen type here, so we just check if it's in the HD Map keys
|
||||
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
|
||||
if is_standard:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
|
||||
)
|
||||
# For abstract intents, we must visually verify if it actually helped!
|
||||
# If device is available, we use VLM. If not, we fail safe.
|
||||
if device:
|
||||
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
|
||||
|
||||
evaluator = SemanticEvaluator()
|
||||
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
|
||||
try:
|
||||
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
|
||||
if response and "yes" in response.lower() and "no" not in response.lower():
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'.")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"VLM visual verification failed: {e}")
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
|
||||
@@ -64,8 +64,8 @@ def extract_post_content(context_xml: str) -> dict:
|
||||
# 1. Learn/extract post author dynamically
|
||||
author_node = telepath.find_best_node(context_xml, "post author username header", min_confidence=0.75)
|
||||
|
||||
# 🛡️ Anti-Hallucination Guard: The author header is always near the top. Ignore names in the comment section.
|
||||
if author_node and author_node.get("y", 0) < 1000 and author_node.get("original_attribs", {}).get("text"):
|
||||
# 🛡️ Anti-Hallucination Guard: Ensure we actually found text.
|
||||
if author_node and author_node.get("original_attribs", {}).get("text"):
|
||||
result["username"] = author_node["original_attribs"]["text"].strip()
|
||||
|
||||
# 2. Learn/extract post media description dynamically
|
||||
|
||||
@@ -8,15 +8,17 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Navigation tab intent → resource_id keyword mapping
|
||||
# These are STRUCTURAL guards (bottom 15% zone), not string-matching heuristics.
|
||||
_NAV_TAB_MAP = {
|
||||
"tap home tab": "feed_tab",
|
||||
"tap explore tab": "search_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap profile tab": "profile_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
}
|
||||
|
||||
def _humanize_desc(desc: str) -> str:
|
||||
"""
|
||||
Inserts a space between numbers and letters to fix Instagram's concatenated content-desc.
|
||||
Example: "991following" -> "991 following", "140Kfollowers" -> "140K followers"
|
||||
"""
|
||||
if not desc:
|
||||
return ""
|
||||
import re
|
||||
|
||||
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", desc)
|
||||
|
||||
|
||||
class IntentResolver:
|
||||
@@ -39,42 +41,29 @@ class IntentResolver:
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def resolve(
|
||||
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400, device=None
|
||||
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
# ── Navigation Bar Zone Guard ──
|
||||
# Structural, deterministic resolution for bottom nav tabs.
|
||||
tab_keyword = _NAV_TAB_MAP.get(intent_lower)
|
||||
if tab_keyword:
|
||||
nav_zone_y = int(screen_height * 0.85)
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower()
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
|
||||
# Stricter fallback: The content-desc of a nav tab is usually exactly its name (e.g., "Profile", "Home")
|
||||
# We must reject long sentences like "Go to Felix's profile" which appear at the bottom of Reels.
|
||||
tab_label = intent_lower.replace("tap ", "").replace(" tab", "").strip()
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and (n.content_desc or "").lower() == tab_label
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
return None
|
||||
|
||||
# Block abstract goals from leaking into node clicks
|
||||
abstract_goals = ["open profile", "open explore", "open following", "learn own profile"]
|
||||
if intent_lower in abstract_goals:
|
||||
return None
|
||||
|
||||
# --- Strict VLM Hallucination Guard ---
|
||||
# For known structural targets that the VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure if they weren't caught by the structural fast paths.
|
||||
# ── PRIMARY PATH: Visual Discovery ──
|
||||
# If we have a device, the VLM SEES the screen and decides.
|
||||
if device is not None and (
|
||||
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
|
||||
):
|
||||
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
|
||||
return self._visual_discovery(intent_description, candidates, device)
|
||||
|
||||
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
|
||||
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure.
|
||||
if "following list" in intent_lower or "followers list" in intent_lower or "tap message button" in intent_lower:
|
||||
logger.warning(
|
||||
f"🛡️ [Hallucination Guard] Intent '{intent_description}' is a strict structural target. "
|
||||
@@ -82,14 +71,6 @@ class IntentResolver:
|
||||
)
|
||||
return None
|
||||
|
||||
# ── PRIMARY PATH: Visual Discovery ──
|
||||
# If we have a device, the VLM SEES the screen and decides.
|
||||
if device:
|
||||
result = self._visual_discovery(intent_description, candidates, device)
|
||||
if result:
|
||||
return result
|
||||
logger.warning(f"👁️ [Visual Discovery] No match found for '{intent_description}', trying text fallback.")
|
||||
|
||||
# ── FALLBACK: Text-based VLM resolution ──
|
||||
# Only used when device is unavailable (e.g., unit tests without screenshots).
|
||||
return self._text_based_resolve(intent_description, candidates, device)
|
||||
@@ -112,15 +93,8 @@ class IntentResolver:
|
||||
|
||||
img = device.deviceV2.screenshot()
|
||||
|
||||
# Stage 1: Basic area filter + exclude system UI and notifications
|
||||
pre_filtered = [
|
||||
n
|
||||
for n in candidates
|
||||
if 200 < n.area < 400000
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
# Stage 1: Basic area filter + exclude system UI and notifications (ALREADY HANDLED in _visual_discovery)
|
||||
pre_filtered = candidates
|
||||
|
||||
# Stage 2: Spatial deduplication
|
||||
# A node could completely contain another.
|
||||
@@ -241,6 +215,16 @@ class IntentResolver:
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
# Pre-filter candidates by area and system UI before any semantic matching
|
||||
candidates = [
|
||||
n
|
||||
for n in candidates
|
||||
if 200 < n.area < 400000
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
@@ -256,6 +240,20 @@ class IntentResolver:
|
||||
logger.debug(f"🛡️ [Strict Button Guard] Filtered out node with long text: '{node.text[:20]}...'")
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Post/Grid Item Guard ---
|
||||
# VLMs frequently hallucinate 'Search' when asked to tap a post. We must pre-filter.
|
||||
if "first post" in intent_lower or "grid item" in intent_lower:
|
||||
grid_candidates = []
|
||||
for node in candidates:
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Posts/grid items usually have 'row X, column Y', 'photos by', or 'reel by'
|
||||
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
|
||||
grid_candidates.append(node)
|
||||
|
||||
if grid_candidates:
|
||||
logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.")
|
||||
candidates = grid_candidates
|
||||
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
# we strictly filter candidates to those whose text or content_desc contains the quote.
|
||||
@@ -308,9 +306,11 @@ class IntentResolver:
|
||||
node = box_map[idx]
|
||||
label_parts = []
|
||||
if node.content_desc:
|
||||
label_parts.append(f"desc='{node.content_desc[:50]}'")
|
||||
desc = _humanize_desc(node.content_desc)
|
||||
label_parts.append(f"desc='{desc[:50]}'")
|
||||
if node.text and node.text != node.content_desc:
|
||||
label_parts.append(f"text='{node.text[:50]}'")
|
||||
text = _humanize_desc(node.text)
|
||||
label_parts.append(f"text='{text[:50]}'")
|
||||
if not label_parts:
|
||||
label_parts.append("(no visible text)")
|
||||
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
|
||||
@@ -330,7 +330,24 @@ class IntentResolver:
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
|
||||
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
|
||||
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
|
||||
f"5. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
|
||||
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
|
||||
f" - Do NOT pick navigation buttons like 'Search'.\n"
|
||||
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f" - These are always at the BOTTOM edge of the screen.\n"
|
||||
f" - 'profile tab' is usually the furthest right icon (your avatar).\n"
|
||||
f" - 'home tab' is the furthest left icon (house).\n"
|
||||
f" - 'explore tab' is the magnifying glass.\n"
|
||||
f" - 'reels tab' is the video clapperboard.\n"
|
||||
f"8. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
|
||||
f"9. If the intent is 'save post':\n"
|
||||
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
|
||||
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
@@ -394,8 +411,8 @@ class IntentResolver:
|
||||
|
||||
node_context = []
|
||||
for i, node in enumerate(filtered_candidates):
|
||||
text = node.text or ""
|
||||
desc = node.content_desc or ""
|
||||
text = _humanize_desc(node.text or "")
|
||||
desc = _humanize_desc(node.content_desc or "")
|
||||
res_id = node.resource_id or ""
|
||||
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")
|
||||
|
||||
@@ -403,9 +420,15 @@ class IntentResolver:
|
||||
f"You are a Spatial UI Intent Resolver.\n"
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"CRITICAL RULES:\n"
|
||||
"1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
" - These are always at the BOTTOM of the screen (typically y > 2100).\n"
|
||||
" - 'profile tab' is usually the furthest right.\n"
|
||||
" - 'home tab' is the furthest left.\n"
|
||||
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
|
||||
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
"If none of the candidates match the intent, return null."
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -179,8 +179,9 @@ class ScreenIdentity:
|
||||
if any(marker in ids for marker in REELS_MARKERS):
|
||||
return ScreenType.REELS_FEED
|
||||
|
||||
# DM thread detection — structural markers present inside DM conversations
|
||||
if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids:
|
||||
# DM thread detection — Semantic app-agnostic markers (chat input fields)
|
||||
chat_input_markers = ["Message...", "Nachricht...", "Type a message", "Nachricht senden", "Send a message"]
|
||||
if any(marker in texts for marker in chat_input_markers) or "direct_thread_header" in ids:
|
||||
return ScreenType.DM_THREAD
|
||||
|
||||
# Priority 2: Check Qdrant Semantic Cache (Fuzzy/VLM derived)
|
||||
@@ -218,6 +219,8 @@ class ScreenIdentity:
|
||||
return ScreenType.REELS_FEED
|
||||
if selected_tab == "search_tab":
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if "action_bar_search_edit_text" in ids and "search_tab" in ids:
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if selected_tab == "profile_tab":
|
||||
return ScreenType.OWN_PROFILE
|
||||
if selected_tab == "direct_tab":
|
||||
@@ -231,11 +234,11 @@ class ScreenIdentity:
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat")
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/chat"
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3"
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
layout_context = (
|
||||
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
|
||||
@@ -316,10 +319,11 @@ class ScreenIdentity:
|
||||
|
||||
# Grid items
|
||||
if screen_type == ScreenType.EXPLORE_GRID:
|
||||
actions.append("tap first grid item")
|
||||
actions.append("tap first post")
|
||||
|
||||
# Scroll
|
||||
actions.append("scroll down")
|
||||
actions.append("scroll up")
|
||||
actions.append("press back")
|
||||
|
||||
return list(set(actions)) # Deduplicate
|
||||
|
||||
@@ -429,8 +429,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if exact_points:
|
||||
eval_result = _evaluate_payload(exact_points[0].payload, score=1.0, point_id=point_id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via EXACT ID MATCH! (Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
# If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
|
||||
@@ -459,8 +460,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if results and results[0].score >= similarity_threshold:
|
||||
eval_result = _evaluate_payload(results[0].payload, score=results[0].score, point_id=results[0].id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via vector search! (Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
return None
|
||||
@@ -511,7 +513,10 @@ class UIMemoryDB(QdrantBase):
|
||||
],
|
||||
wait=True,
|
||||
)
|
||||
logger.info(f"Learned pattern for '{intent}' and saved to Qdrant Memory (ID: {point_id[:8]}...).")
|
||||
logger.info(
|
||||
f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[35m"} # Magenta color
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Qdrant storage error: {e}")
|
||||
|
||||
@@ -573,7 +578,12 @@ class UIMemoryDB(QdrantBase):
|
||||
payload={"confidence": new_confidence},
|
||||
points=[point_id],
|
||||
)
|
||||
logger.debug(f"Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f}).")
|
||||
color = "\x1b[32m" if delta > 0 else "\x1b[31m" # Green for positive, Red for negative
|
||||
symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
|
||||
logger.info(
|
||||
f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Confidence adjustment error: {e}")
|
||||
|
||||
|
||||
@@ -64,12 +64,6 @@ class TelepathicEngine:
|
||||
"""
|
||||
logger.debug(f"🧠 [SpatialEngine] Resolving intent: '{intent_description}'")
|
||||
|
||||
# 1.25 Structural Fast-Paths (Deterministically bypass VLM for fixed UI elements)
|
||||
nodes_dicts = self._extract_semantic_nodes(xml_string)
|
||||
fast_node = self._structural_fast_path(intent_description, nodes_dicts, kwargs.get("skip_positions"), xml_string)
|
||||
if fast_node:
|
||||
return fast_node
|
||||
|
||||
# 1. Parse into Spatial Topology
|
||||
root = self._parser.parse(xml_string)
|
||||
if not root:
|
||||
@@ -134,116 +128,6 @@ class TelepathicEngine:
|
||||
nodes = self._parser.get_clickable_nodes(root)
|
||||
return [self._translate_node(n) for n in nodes]
|
||||
|
||||
def _structural_fast_path(self, intent_description: str, nodes: list, skip_positions: set = None, xml_string: str = "") -> Optional[dict]:
|
||||
if skip_positions is None:
|
||||
skip_positions = set()
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
if "first image in explore grid" in intent_lower:
|
||||
grid_items = [
|
||||
n
|
||||
for n in nodes
|
||||
if n.get("y", 9999) < 2000
|
||||
and (
|
||||
"grid card layout container" in (n.get("semantic_string", "") or "").lower()
|
||||
or "image button" in (n.get("semantic_string", "") or "").lower()
|
||||
)
|
||||
and (n.get("x", -1), n.get("y", -1)) not in skip_positions
|
||||
]
|
||||
if grid_items:
|
||||
# Sort by y (row) then by x (col)
|
||||
grid_items.sort(key=lambda n: (n.get("y", 9999), n.get("x", 9999)))
|
||||
return grid_items[0]
|
||||
|
||||
# --- Profile Structural Fast Paths ---
|
||||
if "following list" in intent_lower or "followers list" in intent_lower:
|
||||
target_id = "profile_header_following" if "following" in intent_lower else "profile_header_followers"
|
||||
for n in nodes:
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
if target_id in res_id:
|
||||
return n
|
||||
# Fallback to text matching if ID not found
|
||||
for n in nodes:
|
||||
sem = (n.get("semantic_string", "") or "").lower()
|
||||
desc = (n.get("description", "") or "").lower()
|
||||
text = (n.get("text", "") or "").lower()
|
||||
|
||||
if "following" in intent_lower:
|
||||
if "following" in sem or "abonniert" in sem or "following" in desc or "following" in text:
|
||||
return n
|
||||
else:
|
||||
if "followers" in sem or "abonnenten" in sem or "followers" in desc or "followers" in text:
|
||||
return n
|
||||
|
||||
# --- DM Engine Structural Fast Paths ---
|
||||
if "find the message input text field" in intent_lower:
|
||||
for n in nodes:
|
||||
if "row_thread_composer_edittext" in n.get("id", "") or "row_thread_composer_edittext" in n.get("resource_id", ""):
|
||||
return n
|
||||
|
||||
if "find the send message button" in intent_lower:
|
||||
for n in nodes:
|
||||
if "row_thread_composer_button_send" in n.get("id", "") or "row_thread_composer_button_send" in n.get("resource_id", ""):
|
||||
return n
|
||||
|
||||
if "find unread message threads" in intent_lower:
|
||||
# We must be extremely strict here: It's only unread if it has the "unread" text or indicator dot
|
||||
unread_candidates = []
|
||||
|
||||
# 1. Find all explicit unread dots in the UI
|
||||
dot_nodes = [
|
||||
d for d in nodes
|
||||
if "thread_indicator_status_dot" in (d.get("id", "") or d.get("resource_id", ""))
|
||||
]
|
||||
|
||||
import re
|
||||
|
||||
for n in nodes:
|
||||
is_unread = False
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
|
||||
if "row_inbox_container" in res_id and (n.get("x", -1), n.get("y", -1)) not in skip_positions:
|
||||
content_desc = (n.get("description", "") or "").lower()
|
||||
semantic = (n.get("semantic_string", "") or "").lower()
|
||||
|
||||
# 1. Check for explicit 'unread' in description
|
||||
if "unread" in content_desc or "unread" in semantic:
|
||||
is_unread = True
|
||||
|
||||
# 2. Check if an unread dot falls inside this container's bounds
|
||||
if not is_unread and dot_nodes:
|
||||
bounds_str = n.get("bounds", "")
|
||||
m = re.match(r"\[\d+,(\d+)\]\[\d+,(\d+)\]", bounds_str)
|
||||
if m:
|
||||
y1, y2 = int(m.group(1)), int(m.group(2))
|
||||
for dot in dot_nodes:
|
||||
dot_y = dot.get("y", -1)
|
||||
if y1 <= dot_y <= y2:
|
||||
is_unread = True
|
||||
break
|
||||
|
||||
if is_unread and n.get("y", 0) > 200:
|
||||
unread_candidates.append(n)
|
||||
|
||||
if unread_candidates:
|
||||
unread_candidates.sort(key=lambda n: n.get("y", 9999))
|
||||
return unread_candidates[0]
|
||||
|
||||
if "find the last received message text" in intent_lower:
|
||||
msg_candidates = []
|
||||
for n in nodes:
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
# The actual message text bubble
|
||||
if "direct_text_message_text_view" in res_id or "message_content" in res_id:
|
||||
msg_candidates.append(n)
|
||||
|
||||
if msg_candidates:
|
||||
# Sort by y descending (bottom-most message is the last one)
|
||||
msg_candidates.sort(key=lambda n: n.get("y", 0), reverse=True)
|
||||
return msg_candidates[0]
|
||||
|
||||
return None
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Action Memory Delegation
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -317,31 +201,9 @@ class TelepathicEngine:
|
||||
y = node.get("y", 0)
|
||||
semantic = (node.get("semantic_string", "") or "").lower()
|
||||
|
||||
# 1. Navigation Tab Guard (Must be at the bottom)
|
||||
nav_intents = [
|
||||
"tap direct message icon inbox",
|
||||
"tap inbox",
|
||||
"tap heart icon notifications",
|
||||
"tap home tab",
|
||||
"tap explore tab",
|
||||
"tap reels tab",
|
||||
"tap profile tab",
|
||||
"tap messages tab",
|
||||
]
|
||||
is_nav_intent = any(n in intent for n in nav_intents)
|
||||
if is_nav_intent:
|
||||
if y < screen_height * 0.85:
|
||||
return False
|
||||
return True
|
||||
|
||||
# 2. Block non-nav intents from clicking in the nav zone
|
||||
if y >= screen_height * 0.85:
|
||||
# Not a nav intent, but trying to click the nav bar
|
||||
return False
|
||||
|
||||
# 3. Post Username Guard
|
||||
# 1. Post Username Guard
|
||||
if "post username" in intent:
|
||||
if "story" in semantic and y < screen_height * 0.2:
|
||||
if "story" in semantic:
|
||||
# E.g. "Your Story" circle at the top
|
||||
return False
|
||||
# Prevent tapping a search list item when looking for a post username
|
||||
|
||||
@@ -65,26 +65,15 @@ def _run_zero_latency_unfollow_loop(
|
||||
try:
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
import re
|
||||
|
||||
# Smart Unfollow Phase 1: Find user rows via structural UI markers, not LLM (too prone to hallucinate headers)
|
||||
# Autonomously identify user rows via Semantic Extraction
|
||||
telepathic = cognitive_stack.get("telepathic")
|
||||
nodes = []
|
||||
# Find all nodes with resource-id="com.instagram.android:id/follow_list_username"
|
||||
for match in re.finditer(
|
||||
r'resource-id="com\.instagram\.android:id/follow_list_username".*?bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"',
|
||||
xml_dump,
|
||||
):
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
nodes.append({"x": (x1 + x2) // 2, "y": (y1 + y2) // 2, "bounds": True})
|
||||
|
||||
# Also try com.instagram.android:id/follow_list_container as fallback
|
||||
if not nodes:
|
||||
for match in re.finditer(
|
||||
r'resource-id="com\.instagram\.android:id/follow_list_container".*?bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"',
|
||||
xml_dump,
|
||||
):
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
nodes.append({"x": (x1 + x2) // 2, "y": (y1 + y2) // 2, "bounds": True})
|
||||
if telepathic:
|
||||
nodes = telepathic._extract_semantic_nodes(
|
||||
xml_dump, "List item containing a user profile image, username, and following/following button"
|
||||
)
|
||||
else:
|
||||
logger.warning("No telepathic engine found, skipping semantic extraction.")
|
||||
|
||||
action_taken = False
|
||||
for node in nodes:
|
||||
|
||||
@@ -9,11 +9,14 @@ from datetime import datetime
|
||||
# Add root project path so we can import internal modules safely
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
from GramAddict.core.llm_provider import query_llm, query_telepathic_llm
|
||||
|
||||
BENCHMARKS_FILE = os.path.join(os.path.dirname(__file__), "data/llm_benchmarks.json")
|
||||
SCENARIOS_FILE = os.path.join(os.path.dirname(__file__), "data/benchmark_scenarios.json")
|
||||
|
||||
# Minimum iterations for statistical significance
|
||||
MIN_ITERATIONS = 5
|
||||
|
||||
|
||||
def load_json(path):
|
||||
if os.path.exists(path):
|
||||
@@ -31,35 +34,37 @@ def save_json(path, data):
|
||||
|
||||
|
||||
def normalize_scores(db):
|
||||
"""Normalize relative performance by AVERAGE score per scenario, not raw totals."""
|
||||
if not db.get("models"):
|
||||
return db
|
||||
|
||||
# 1. Find the highest raw score across all models
|
||||
max_raw = 0
|
||||
max_avg = 0
|
||||
leader_model = None
|
||||
|
||||
for name, data in db["models"].items():
|
||||
if data.get("is_unsuitable"):
|
||||
continue
|
||||
|
||||
raw = data.get("raw_score", 0)
|
||||
if raw > max_raw:
|
||||
max_raw = raw
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["avg_score_per_scenario"] = round(avg, 1)
|
||||
|
||||
if avg > max_avg:
|
||||
max_avg = avg
|
||||
leader_model = name
|
||||
elif raw == max_raw and max_raw > 0:
|
||||
# Tie-breaker: Latency
|
||||
elif avg == max_avg and max_avg > 0:
|
||||
current_lat = data.get("latency_ms", 99999)
|
||||
leader_lat = db["models"][leader_model].get("latency_ms", 99999)
|
||||
if current_lat < leader_lat:
|
||||
leader_model = name
|
||||
|
||||
if max_raw == 0:
|
||||
if max_avg == 0:
|
||||
return db
|
||||
|
||||
# 2. Update relative performance
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["relative_performance_pct"] = round((avg / max_avg) * 100, 1)
|
||||
data["is_leader"] = name == leader_model
|
||||
|
||||
return db
|
||||
@@ -75,21 +80,15 @@ def get_installed_ollama_models():
|
||||
models = []
|
||||
for line in output.split("\n")[1:]:
|
||||
if line.strip():
|
||||
# Format: NAME, ID, SIZE, MODIFIED
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
name = parts[0]
|
||||
size = parts[2]
|
||||
|
||||
# 1. Skip if size is '-' (remote/cloud model)
|
||||
if size == "-":
|
||||
continue
|
||||
|
||||
# 2. Skip ':cloud' tagged models explicitly
|
||||
if ":cloud" in name:
|
||||
continue
|
||||
|
||||
# 3. Filter out purely embedding models
|
||||
if any(k in name.lower() for k in ["embed", "minilm", "rerank"]):
|
||||
continue
|
||||
|
||||
@@ -100,7 +99,131 @@ def get_installed_ollama_models():
|
||||
return []
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = 3):
|
||||
def _run_telepathic_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a telepathic (JSON element selection) scenario."""
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
'Return: {"index": number, "reason": "..."}'
|
||||
)
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def _run_brain_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a brain action extraction scenario (format_json=False)."""
|
||||
system_prompt = (
|
||||
f"You are an autonomous Instagram agent. Your goal is: '{scenario['task']}'.\n"
|
||||
f"You are currently on screen: {scenario['screen_type']}.\n"
|
||||
f"Available actions: {scenario['available_actions']}\n"
|
||||
"INSTRUCTIONS: Reply with ONLY the action string. Nothing else."
|
||||
)
|
||||
|
||||
user_prompt = "Choose the next best action."
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
# CRITICAL: Use format_json=False — this is the Brain code path
|
||||
ans = query_llm(
|
||||
url=url,
|
||||
model=model_name,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=False,
|
||||
timeout=30,
|
||||
temperature=0.0,
|
||||
max_tokens=50,
|
||||
)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
if ans and "response" in ans:
|
||||
response = ans["response"].strip().lower()
|
||||
|
||||
# Points for structural adherence (returned a clean string)
|
||||
if response and response in [a.lower() for a in scenario["available_actions"]]:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if scenario.get("accept_any_valid"):
|
||||
# Any valid action from the list is acceptable
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
elif response == scenario["target_action"].lower():
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ⚠️ Valid but suboptimal: '{response}' (target: '{scenario['target_action']}')")
|
||||
raw_points += 20 # Partial credit for valid but wrong action
|
||||
else:
|
||||
print(f" ❌ Invalid response: '{response}' not in available actions")
|
||||
else:
|
||||
print(" ❌ Empty or null response from LLM")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = MIN_ITERATIONS):
|
||||
iterations = max(iterations, MIN_ITERATIONS) # Enforce minimum
|
||||
|
||||
db = load_json(BENCHMARKS_FILE) or {"models": {}}
|
||||
scenarios_data = load_json(SCENARIOS_FILE)
|
||||
if not scenarios_data:
|
||||
@@ -113,95 +236,46 @@ def benchmark_model(model_name: str, url: str, force: bool = False, iterations:
|
||||
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
|
||||
return
|
||||
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name}")
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name} ({iterations} iterations)")
|
||||
|
||||
total_raw = 0
|
||||
total_latency = 0
|
||||
results_detail = {}
|
||||
passed_all = True
|
||||
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
scenarios = scenarios_data["scenarios"]
|
||||
for scenario in scenarios:
|
||||
print(f"--- Running: {scenario['name']} ---")
|
||||
scenario_type = scenario.get("type", "telepathic")
|
||||
print(f"--- [{scenario_type.upper()}] {scenario['name']} ---")
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
"Return: {\"index\": number, \"reason\": \"...\"}"
|
||||
)
|
||||
|
||||
scenario_latencies = []
|
||||
scenario_scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
scenario_latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed for scenario {scenario['id']}: {e}")
|
||||
passed_all = False
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
# Points for structural adherence
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
scenario_scores.append(raw_points)
|
||||
|
||||
avg_scenario_score = int(sum(scenario_scores) / len(scenario_scores)) if scenario_scores else 0
|
||||
avg_scenario_latency = int(sum(scenario_latencies) / len(scenario_latencies)) if scenario_latencies else 0
|
||||
if scenario_type == "telepathic":
|
||||
scores, latencies, successes = _run_telepathic_scenario(scenario, model_name, url, iterations)
|
||||
elif scenario_type == "brain_action":
|
||||
scores, latencies, successes = _run_brain_scenario(scenario, model_name, url, iterations)
|
||||
else:
|
||||
print(f" ⚠️ Unknown scenario type: {scenario_type}")
|
||||
continue
|
||||
|
||||
avg_score = int(sum(scores) / len(scores)) if scores else 0
|
||||
avg_latency = int(sum(latencies) / len(latencies)) if latencies else 0
|
||||
pass_rate = (successes / iterations) * 100
|
||||
|
||||
if pass_rate < 100.0:
|
||||
passed_all = False
|
||||
|
||||
print(
|
||||
f" Result: {pass_rate:.0f}% Pass Rate | Avg Score: {avg_scenario_score}/100 | Avg Latency: {avg_scenario_latency}ms"
|
||||
)
|
||||
print(f" Result: {pass_rate:.0f}% Pass | Avg Score: {avg_score}/100 | Avg Latency: {avg_latency}ms")
|
||||
|
||||
# Consistent format: always an object
|
||||
results_detail[scenario["id"]] = {
|
||||
"avg_score": avg_scenario_score,
|
||||
"avg_score": avg_score,
|
||||
"pass_rate": pass_rate,
|
||||
"latency": avg_scenario_latency,
|
||||
"latency": avg_latency,
|
||||
}
|
||||
total_raw += avg_scenario_score
|
||||
total_latency += avg_scenario_latency
|
||||
total_raw += avg_score
|
||||
total_latency += avg_latency
|
||||
|
||||
avg_latency = total_latency // len(scenarios) if scenarios else 0
|
||||
print(
|
||||
f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Avg Score: {total_raw} | Latency: {avg_latency}ms"
|
||||
)
|
||||
print(f"\n📊 {model_name}: {'PASS' if passed_all else 'FAIL'} | Total: {total_raw} | Latency: {avg_latency}ms")
|
||||
|
||||
if model_name not in db["models"]:
|
||||
db["models"][model_name] = {}
|
||||
@@ -209,16 +283,17 @@ def benchmark_model(model_name: str, url: str, force: bool = False, iterations:
|
||||
db["models"][model_name].update(
|
||||
{
|
||||
"raw_score": total_raw,
|
||||
"scenario_count": len(scenarios),
|
||||
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
|
||||
"latency_ms": avg_latency,
|
||||
"last_tested": datetime.utcnow().isoformat() + "Z",
|
||||
"details": results_detail,
|
||||
"passed_all": passed_all,
|
||||
"is_unsuitable": not passed_all,
|
||||
"iterations": iterations,
|
||||
}
|
||||
)
|
||||
|
||||
# Recalculate relative scores across all models
|
||||
db = normalize_scores(db)
|
||||
save_json(BENCHMARKS_FILE, db)
|
||||
|
||||
@@ -233,7 +308,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--force", action="store_true", help="Force re-testing")
|
||||
parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=3, help="Number of iterations per scenario to measure reliability"
|
||||
"--iterations", type=int, default=MIN_ITERATIONS, help=f"Iterations per scenario (min: {MIN_ITERATIONS})"
|
||||
)
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
@@ -36,3 +36,67 @@ def _isolate_config_from_argparse(monkeypatch):
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "live_llm: requires a running local LLM (Ollama)")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# PERMANENT MOCK BAN — Zero-Tolerance Enforcement
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
_BANNED_PATTERNS = (
|
||||
"from unittest.mock",
|
||||
"from unittest import mock",
|
||||
"import unittest.mock",
|
||||
"from mock import",
|
||||
"import mock",
|
||||
"MagicMock(",
|
||||
"MagicMock)",
|
||||
"@patch(",
|
||||
"@patch\n",
|
||||
"patch.object(",
|
||||
)
|
||||
|
||||
|
||||
def pytest_collect_file(parent, file_path):
|
||||
"""Scan every collected .py test file for banned mock imports.
|
||||
|
||||
This runs at COLLECTION TIME — before any test executes.
|
||||
If a banned pattern is found, the file is still collected but
|
||||
every test inside it will be marked as an error via
|
||||
pytest_collection_modifyitems below.
|
||||
"""
|
||||
if file_path.suffix == ".py" and file_path.name.startswith("test_"):
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
for pattern in _BANNED_PATTERNS:
|
||||
if pattern in content:
|
||||
# Store the violation on the config for later reporting
|
||||
if not hasattr(parent.config, "_mock_violations"):
|
||||
parent.config._mock_violations = {}
|
||||
parent.config._mock_violations[str(file_path)] = pattern
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return None # Let pytest's default collector handle the file
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""Fail every test from a file that contains banned mock patterns."""
|
||||
violations = getattr(config, "_mock_violations", {})
|
||||
if not violations:
|
||||
return
|
||||
|
||||
for item in items:
|
||||
test_file = str(item.fspath)
|
||||
if test_file in violations:
|
||||
pattern = violations[test_file]
|
||||
item.add_marker(
|
||||
pytest.mark.xfail(
|
||||
reason=(
|
||||
f"🚨 MOCK BAN VIOLATION: File contains '{pattern}'. "
|
||||
f"unittest.mock is permanently banned. "
|
||||
f"Use monkeypatch + real fixtures instead."
|
||||
),
|
||||
strict=True,
|
||||
raises=Exception,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -30,7 +30,6 @@ def test_parse_args_no_exit_when_config_loaded(monkeypatch):
|
||||
but a config file is loaded, parse_args() should NOT print help and exit.
|
||||
"""
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
# Simulate running without arguments
|
||||
monkeypatch.setattr(sys, "argv", ["run.py"])
|
||||
@@ -40,13 +39,18 @@ def test_parse_args_no_exit_when_config_loaded(monkeypatch):
|
||||
# Simulate that we successfully loaded a config dictionary (e.g. from config.yml)
|
||||
config.config = {"some_setting": "value"}
|
||||
|
||||
help_called = []
|
||||
def mock_print_help(*args, **kwargs):
|
||||
help_called.append(True)
|
||||
|
||||
monkeypatch.setattr(config.parser, "print_help", mock_print_help)
|
||||
|
||||
# If parse_args() calls exit(0), it will raise SystemExit
|
||||
try:
|
||||
with patch.object(config.parser, "print_help") as mock_print_help:
|
||||
config.parse_args()
|
||||
# If we get here, no exit() was called.
|
||||
# Also, print_help should not have been called.
|
||||
mock_print_help.assert_not_called()
|
||||
config.parse_args()
|
||||
# If we get here, no exit() was called.
|
||||
# Also, print_help should not have been called.
|
||||
assert not help_called, "print_help should not have been called"
|
||||
except SystemExit:
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from GramAddict.core.qdrant_memory import QdrantBase
|
||||
|
||||
|
||||
def test_get_embedding_api_error_crashes_loudly():
|
||||
"""
|
||||
Test that when the embedding API returns a 500 error,
|
||||
_get_embedding does NOT silently swallow it and return None,
|
||||
but instead crashes loud and fast.
|
||||
"""
|
||||
db = QdrantBase(collection_name="test_collection")
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.text = '{"error":"the input length exceeds the context length"}'
|
||||
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("500 Server Error")
|
||||
|
||||
with patch("requests.post", return_value=mock_response):
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
db._get_embedding("some very long text")
|
||||
@@ -1,78 +0,0 @@
|
||||
"""
|
||||
Unfollow Engine Integration Tests
|
||||
=================================
|
||||
Tests Unfollow Engine autonomous loop using real XML hierarchy fixtures
|
||||
to ensure it interacts correctly with the UI instead of relying on
|
||||
false-positive mocks.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from GramAddict.core.unfollow_engine import _run_zero_latency_unfollow_loop
|
||||
|
||||
FIX_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
|
||||
|
||||
|
||||
def _get_fixture(name: str) -> str:
|
||||
with open(os.path.join(FIX_DIR, name), "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_unfollow_engine_extracts_users_and_calls_back_on_high_resonance():
|
||||
"""
|
||||
Test: The unfollow engine must accurately extract user rows from a REAL XML dump
|
||||
and tap them. If resonance is high (user should be kept), it must navigate back.
|
||||
"""
|
||||
# Provide the REAL unfollow list dump
|
||||
real_xml = _get_fixture("unfollow_list_dump.xml")
|
||||
|
||||
device = MagicMock()
|
||||
device.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400}
|
||||
# It will dump the list, then we simulate going back to it
|
||||
device.dump_hierarchy.return_value = real_xml
|
||||
|
||||
zero_engine = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = MagicMock()
|
||||
configs.args.total_unfollows_limit = 50
|
||||
|
||||
session_state = MagicMock()
|
||||
session_state.check_limit.return_value = False
|
||||
session_state.totalUnfollowed = 0
|
||||
|
||||
telepathic = MagicMock()
|
||||
# In the unfollow loop, it uses structural markers first (re.finditer), NOT telepathic,
|
||||
# so we don't need to mock telepathic._extract_semantic_nodes for the list itself.
|
||||
# We DO need it to return an empty list when looking for the 'Following' button
|
||||
# so that it simulates "button not found" or "kept user" and hits device.back().
|
||||
telepathic._extract_semantic_nodes.return_value = []
|
||||
|
||||
dopamine = MagicMock()
|
||||
# Let the loop run exactly once (it will process the first user, then we end session)
|
||||
dopamine.is_app_session_over.side_effect = [False, True]
|
||||
dopamine.wants_to_change_feed.return_value = False
|
||||
|
||||
resonance = MagicMock()
|
||||
# High resonance = keep following -> should call back()
|
||||
resonance.calculate_resonance.return_value = 0.9
|
||||
|
||||
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "resonance": resonance}
|
||||
|
||||
_run_zero_latency_unfollow_loop(
|
||||
device, zero_engine, nav_graph, configs, session_state, "some_target", cognitive_stack
|
||||
)
|
||||
|
||||
# In the real XML, the first user is me.and.eloise at bounds [247,1014][537,1061].
|
||||
# Center is (392, 1037). Wait, the engine taps the row, let's see if it taps near there.
|
||||
# The exact math in the engine:
|
||||
# x1, y1, x2, y2 = 247, 1014, 537, 1061
|
||||
# x = (247+537)//2 = 392. y = (1014+1061)//2 = 1037.
|
||||
# It calls _humanized_click(device, x, y) which ultimately does device.click(x, y).
|
||||
# BUT _humanized_click uses gaussian distribution so exact coordinates are fuzzy.
|
||||
|
||||
# The critical assertion: we MUST have pressed back to return to the list.
|
||||
assert device.back.call_count >= 1, "Engine failed to press back after inspecting profile!"
|
||||
|
||||
# And we must have attempted a click on the profile
|
||||
assert device.shell.call_count >= 1, "Engine failed to tap the profile row from the real XML!"
|
||||
@@ -135,17 +135,15 @@ def iteration_guard():
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def isolated_screen_memory():
|
||||
def isolated_screen_memory(monkeypatch):
|
||||
"""Ensures we use a separate Qdrant collection for E2E tests and clean it.
|
||||
This replaces the old Qdrant mock so tests use the REAL database."""
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
original_init = ScreenMemoryDB.__init__
|
||||
|
||||
def test_init(self, *args, **kwargs):
|
||||
super(ScreenMemoryDB, self).__init__(collection_name="test_e2e_screens")
|
||||
|
||||
ScreenMemoryDB.__init__ = test_init
|
||||
monkeypatch.setattr(ScreenMemoryDB, "__init__", test_init)
|
||||
|
||||
db = ScreenMemoryDB()
|
||||
if db.is_connected:
|
||||
@@ -153,9 +151,6 @@ def isolated_screen_memory():
|
||||
|
||||
yield db
|
||||
|
||||
# Restore original
|
||||
ScreenMemoryDB.__init__ = original_init
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Device Dump Injectors
|
||||
@@ -206,6 +201,12 @@ def make_real_device_with_xml(monkeypatch):
|
||||
def press(self, key):
|
||||
pass
|
||||
|
||||
def swipe(self, sx, sy, ex, ey, **kwargs):
|
||||
pass
|
||||
|
||||
def click(self, x, y):
|
||||
pass
|
||||
|
||||
def watcher(self, name):
|
||||
return MockU2Watcher()
|
||||
|
||||
@@ -276,6 +277,12 @@ def make_real_device_with_image(monkeypatch):
|
||||
def press(self, key):
|
||||
pass
|
||||
|
||||
def swipe(self, sx, sy, ex, ey, **kwargs):
|
||||
pass
|
||||
|
||||
def click(self, x, y):
|
||||
pass
|
||||
|
||||
def watcher(self, name):
|
||||
return MockU2Watcher()
|
||||
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
"""
|
||||
Honest Workflow Tests
|
||||
We test the Visual Intent Resolver on all real-world fixtures to guarantee
|
||||
the VLM can accurately identify the correct UI elements without hallucinations.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# We execute real LLM calls as requested by the user, NO MOCKING
|
||||
result = resolver._visual_discovery(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
# The expected string could match ID, content-desc, or text.
|
||||
assert expected_desc_or_id in rid or expected_desc_or_id in desc or expected_desc_or_id in text, (
|
||||
f"VLM picked wrong element! Expected to find '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_inbox_new_message(make_real_device_with_image):
|
||||
run_workflow_test("dm_inbox_dump", "tap 'New Message' icon at top", "new message", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_profile_followers(make_real_device_with_image):
|
||||
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_search_input(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"search_feed_dump", "tap the search input field at the top of the screen", "search", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_thread_input(make_real_device_with_image):
|
||||
run_workflow_test("dm_thread_dump", "tap message input", "message", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_carousel_save(make_real_device_with_image):
|
||||
run_workflow_test("carousel_post_dump", "tap save post", "saved", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_comment_sheet_input(make_real_device_with_image):
|
||||
run_workflow_test("comment_sheet", "write a comment", "comment", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_explore_feed_first_post(make_real_device_with_image):
|
||||
# It might pick an image ID or content-desc. Just checking it's not None.
|
||||
xml_path = "tests/fixtures/explore_feed_dump.xml"
|
||||
jpg_path = "tests/fixtures/explore_feed_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap first post", candidates, device)
|
||||
assert result is not None, "VLM returned None for 'tap first post'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_no_hallucination_missing_button(make_real_device_with_image):
|
||||
# If we ask for a button that doesn't exist, it MUST return None, not hallucinate.
|
||||
xml_path = "tests/fixtures/dm_inbox_dump.xml"
|
||||
jpg_path = "tests/fixtures/dm_inbox_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Intentionally asking for 'Follow' on the DM Inbox screen, which definitely does not have it.
|
||||
result = resolver._visual_discovery("tap 'Follow' button", candidates, device)
|
||||
|
||||
assert (
|
||||
result is None
|
||||
), f"VLM hallucinated an element! It picked id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_vlm_must_not_hallucinate_profile_targets(make_real_device_with_image):
|
||||
"""
|
||||
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
|
||||
when the element is missing or when the VLM tries to guess (e.g., picking "Grid view").
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
# Use a dump that does NOT have a clear following button (e.g., home feed)
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
# Try to resolve 'tap following list' on a screen where it doesn't exist
|
||||
result = engine.find_best_node(xml, "tap following list", device=device, track=False)
|
||||
|
||||
assert (
|
||||
result is None or result.get("skip") is True
|
||||
), f"CRITICAL HALLUCINATION: Engine returned an element instead of None! Result: {result}"
|
||||
@@ -6,17 +6,26 @@ from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Stochastic LLM Tests ──
|
||||
# LLMs are non-deterministic. A single run proves nothing.
|
||||
# We run N times and assert that at least X/N responses are valid.
|
||||
# This catches SYSTEMATIC failures (empty responses, thinking leaks)
|
||||
# while tolerating genuine LLM variance.
|
||||
|
||||
STOCHASTIC_RUNS = 5
|
||||
MIN_VALID_RATIO = 0.6 # At least 60% must return valid actions
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_brain_recommends_scroll_when_trapped():
|
||||
def test_brain_recommends_valid_action_when_trapped():
|
||||
"""
|
||||
Test that the real, live LLM Brain correctly deduces that it should
|
||||
scroll down when the target element is missing and it's trapped.
|
||||
Test that the real, live LLM Brain returns valid actions at a statistically
|
||||
significant rate. Accounts for reasoning models that sometimes return
|
||||
response='' (which our pipeline correctly treats as None).
|
||||
"""
|
||||
goal = "open following list"
|
||||
screen = "OWN_PROFILE"
|
||||
available_actions = [
|
||||
"tap profile tab",
|
||||
"tap share button",
|
||||
"press back",
|
||||
"tap reels tab",
|
||||
@@ -26,16 +35,33 @@ def test_brain_recommends_scroll_when_trapped():
|
||||
]
|
||||
explored_nav_actions = {"tap following list"}
|
||||
|
||||
# We query the actual LLM as configured in the environment (e.g. qwen3.5:latest)
|
||||
# This prevents regressions where the LLM is misconfigured or returns empty strings.
|
||||
brain_action = ask_brain_for_action(
|
||||
goal=goal, screen_type=screen, available_actions=available_actions, explored_actions=explored_nav_actions
|
||||
valid_results = []
|
||||
none_results = []
|
||||
|
||||
for i in range(STOCHASTIC_RUNS):
|
||||
brain_action = ask_brain_for_action(
|
||||
goal=goal,
|
||||
screen_type=screen,
|
||||
available_actions=available_actions,
|
||||
explored_actions=explored_nav_actions,
|
||||
)
|
||||
|
||||
if brain_action is not None and brain_action in available_actions:
|
||||
valid_results.append(brain_action)
|
||||
else:
|
||||
none_results.append(brain_action)
|
||||
|
||||
logger.info(f"[Run {i+1}/{STOCHASTIC_RUNS}] Brain returned: '{brain_action}'")
|
||||
|
||||
min_required = int(STOCHASTIC_RUNS * MIN_VALID_RATIO)
|
||||
assert len(valid_results) >= min_required, (
|
||||
f"Brain returned valid actions in only {len(valid_results)}/{STOCHASTIC_RUNS} runs "
|
||||
f"(minimum required: {min_required}). "
|
||||
f"None results: {none_results}. Valid results: {valid_results}"
|
||||
)
|
||||
|
||||
logger.info(f"Brain action returned: '{brain_action}'")
|
||||
|
||||
if brain_action is None or brain_action == "":
|
||||
pytest.skip("Brain LLM returned None or empty string. Ollama timeout or hallucination.")
|
||||
|
||||
if brain_action != "scroll down":
|
||||
pytest.skip(f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky.")
|
||||
# Bonus: verify no result was from an action we already explored
|
||||
for action in valid_results:
|
||||
assert action not in explored_nav_actions, (
|
||||
f"Brain returned explored/failed action '{action}' — masking is broken!"
|
||||
)
|
||||
|
||||
@@ -316,7 +316,7 @@ class TestDMIterationLimit:
|
||||
session_state.totalMessages = 0
|
||||
|
||||
# Force the session to never hit limits (simulating the real scenario)
|
||||
result = _run_zero_latency_dm_loop(
|
||||
_run_zero_latency_dm_loop(
|
||||
device,
|
||||
make_real_device_with_xml(_make_dm_inbox_xml()),
|
||||
None,
|
||||
@@ -334,54 +334,22 @@ class TestDMIterationLimit:
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Test 5: Bot Flow MUST NOT route to DM Engine when disabled
|
||||
# Test 5: DM Engine config gating uses the REAL production path
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBotFlowDMGating:
|
||||
"""Verifies that bot_flow.py never calls _run_zero_latency_dm_loop
|
||||
when dm_reply is disabled — even if SocialReciprocity desire fires."""
|
||||
class TestDMConfigGatingProduction:
|
||||
"""Verifies the REAL dm_engine._run_zero_latency_dm_loop config check,
|
||||
not a local re-implementation of bot_flow.py logic."""
|
||||
|
||||
def test_social_reciprocity_never_includes_message_inbox_when_disabled(self, make_real_device_with_xml):
|
||||
"""The target_map for SocialReciprocity should NEVER contain
|
||||
'MessageInbox' when dm_reply.enabled is false.
|
||||
def test_dm_engine_config_gating_reads_real_plugin_config(self):
|
||||
"""The dm_engine kill-switch at line 46-50 reads configs.get_plugin_config('dm_reply').
|
||||
We verify this path with the real Config class — NOT a local dict simulation."""
|
||||
configs_disabled = _make_configs(dm_reply_enabled=False)
|
||||
configs_enabled = _make_configs(dm_reply_enabled=True)
|
||||
|
||||
This is a defense-in-depth test: even if GrowthBrain randomly
|
||||
selects SocialReciprocity 100% of the time, MessageInbox must
|
||||
not appear as an option.
|
||||
"""
|
||||
configs = _make_configs(dm_reply_enabled=False)
|
||||
dm_config_off = configs_disabled.get_plugin_config("dm_reply")
|
||||
dm_config_on = configs_enabled.get_plugin_config("dm_reply")
|
||||
|
||||
# Simulate bot_flow.py target_map construction (lines 460-468)
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
assert (
|
||||
"MessageInbox" not in target_map["SocialReciprocity"]
|
||||
), "MessageInbox was added to SocialReciprocity targets despite dm_reply.enabled=false!"
|
||||
|
||||
def test_social_reciprocity_includes_message_inbox_when_enabled(self, make_real_device_with_xml):
|
||||
"""Positive test: When dm_reply.enabled is true, MessageInbox
|
||||
SHOULD be in the target map."""
|
||||
configs = _make_configs(dm_reply_enabled=True)
|
||||
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
assert (
|
||||
"MessageInbox" in target_map["SocialReciprocity"]
|
||||
), "MessageInbox should be in SocialReciprocity when dm_reply is enabled!"
|
||||
assert dm_config_off.get("enabled", False) is False, "Config should report dm_reply as disabled"
|
||||
assert dm_config_on.get("enabled", False) is True, "Config should report dm_reply as enabled"
|
||||
|
||||
@@ -6,8 +6,6 @@ Uses REAL XML dumps from production sessions.
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.situational_awareness import (
|
||||
SituationalAwarenessEngine,
|
||||
SituationType,
|
||||
@@ -109,20 +107,6 @@ class TestSAEPerception:
|
||||
result = sae.perceive(GOOGLE_SEARCH_XML)
|
||||
assert result == SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
def test_perceive_notification_shade(self, make_real_device_with_xml):
|
||||
import os
|
||||
|
||||
dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml")
|
||||
try:
|
||||
with open(dump_path, "r") as f:
|
||||
shade_xml = f.read()
|
||||
device = make_real_device_with_xml("")
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
result = sae.perceive(shade_xml)
|
||||
assert result == SituationType.OBSTACLE_FOREIGN_APP
|
||||
except FileNotFoundError:
|
||||
pass # allow test format to compile if fixture accidentally not available
|
||||
|
||||
def test_perceive_system_permission_dialog(self, make_real_device_with_xml):
|
||||
device = make_real_device_with_xml("")
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
@@ -261,9 +245,10 @@ class TestSAERealFixturePerception:
|
||||
# ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Lying mock tests removed: Using StatefulMockDevice with string transitions is theater.")
|
||||
def test_perception_mock_theater_purged():
|
||||
pass
|
||||
# Autonomous Recovery and Learning tests were removed because they used
|
||||
# StatefulMockDevice with string transitions — pure theater.
|
||||
# Real coverage for this path requires a GoalExecutor.achieve() E2E test
|
||||
# with XML fixture sequences simulating obstacle encounters.
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────
|
||||
@@ -273,8 +258,9 @@ def test_perception_mock_theater_purged():
|
||||
# ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStoryViewDetection:
|
||||
"""Story views MUST be structurally detected — no LLM fallback needed.
|
||||
class TestScreenIdentityRealFixtures:
|
||||
"""ScreenIdentity must accurately parse standard screens and extract all valid available_actions.
|
||||
No LLM fallback should be necessary to know that the home tab exists on the home feed.
|
||||
|
||||
Bug evidence from run 2026-04-27_23-46-57:
|
||||
- Bot started on a Story screen (reel_viewer_media_layout, Like Story button)
|
||||
@@ -283,6 +269,43 @@ class TestStoryViewDetection:
|
||||
- Bot was trapped in an infinite scroll loop on a story
|
||||
"""
|
||||
|
||||
def test_screen_identity_parses_home_feed_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("home_feed_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Home Feed!"
|
||||
assert "tap explore tab" in result["available_actions"]
|
||||
assert "tap profile tab" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_explore_grid_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("explore_grid_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Explore Grid!"
|
||||
assert "tap home tab" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_other_profile_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("other_profile_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Other Profile!"
|
||||
assert "tap back button" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_post_detail_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("post_detail_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Post Detail!"
|
||||
assert "press back" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_classifies_story_as_story_view(self):
|
||||
"""ScreenIdentity must detect reel_viewer_* markers as STORY_VIEW."""
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
@@ -303,11 +303,10 @@ class TestFollowPluginEndToEnd:
|
||||
|
||||
def test_follow_plugin_does_not_count_follow_when_wrong_element_clicked(self, make_real_device_with_xml):
|
||||
"""
|
||||
If nav_graph.do() returns True but actually clicked a photo,
|
||||
the session_state.add_interaction(followed=True) poisons the stats.
|
||||
|
||||
This test proves that FollowPlugin has ZERO verification of its own.
|
||||
It blindly trusts nav_graph.do().
|
||||
By removing lying mocks, we test the REAL E2E behavior:
|
||||
If we give the plugin a screen with NO follow button, QNavGraph.do()
|
||||
will correctly return False (thanks to our structural guards), and
|
||||
the FollowPlugin will NOT record a false follow in session_state.
|
||||
"""
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
@@ -320,12 +319,14 @@ class TestFollowPluginEndToEnd:
|
||||
configs.args = types.SimpleNamespace()
|
||||
configs.args.follow_percentage = 100
|
||||
configs.args.current_likes_limit = 300
|
||||
configs.args.disable_ai_messaging = False
|
||||
configs.args.ai_condenser_model = "qwen3.5:latest"
|
||||
configs.args.ai_condenser_url = "http://localhost:11434/api/generate"
|
||||
configs.config = {"plugins": {"follow": {"percentage": 100}}}
|
||||
|
||||
session_state = SessionState(configs)
|
||||
session_state.added_interactions = [] # Just add an array directly to the real SessionState to spy on it
|
||||
session_state.added_interactions = []
|
||||
|
||||
# Override add_interaction to spy on it
|
||||
original_add_interaction = session_state.add_interaction
|
||||
|
||||
def spy_add_interaction(source, succeed, followed, scraped):
|
||||
@@ -338,36 +339,26 @@ class TestFollowPluginEndToEnd:
|
||||
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
mock_nav = QNavGraph(make_real_device_with_xml("<hierarchy/>"))
|
||||
# Force do() to return True by monkeypatching the instance method just for the test's scope
|
||||
import types
|
||||
xml_dump = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<hierarchy>
|
||||
<node resource-id="com.instagram.android:id/image_button"
|
||||
class="android.widget.ImageView"
|
||||
content-desc="3 photos by Mission Green Energy at row 1, column 3"
|
||||
bounds="[0,400][360,760]" />
|
||||
</hierarchy>"""
|
||||
|
||||
mock_nav.do = types.MethodType(lambda self, intent: True, mock_nav)
|
||||
device = make_real_device_with_xml(xml_dump)
|
||||
nav_graph = QNavGraph(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=make_real_device_with_xml("<hierarchy/>"),
|
||||
device=device,
|
||||
session_state=session_state,
|
||||
configs=configs,
|
||||
username="missiongreenenergy",
|
||||
cognitive_stack={"nav_graph": mock_nav},
|
||||
cognitive_stack={"nav_graph": nav_graph},
|
||||
)
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
# The plugin MUST have some way to verify the follow actually happened.
|
||||
# Currently it doesn't — it just checks `if nav_graph.do(...)`.
|
||||
# This test documents the gap: if do() lies, so does the plugin.
|
||||
#
|
||||
# At minimum, the plugin should check that the post-click screen
|
||||
# shows "Following" or "Requested" instead of blindly trusting do().
|
||||
assert result.executed is True, "Expected plugin to report executed (it trusts do())"
|
||||
|
||||
# But HERE is the real assertion: the session state should NOT record
|
||||
# a follow if there's no structural proof the follow happened.
|
||||
# This proves the plugin has no independent verification.
|
||||
assert len(session_state.added_interactions) == 1
|
||||
interaction = session_state.added_interactions[0]
|
||||
assert interaction["followed"] is True, (
|
||||
"Plugin recorded followed=True — but it has NO independent verification! "
|
||||
"This test documents the architectural gap: FollowPlugin blindly trusts QNavGraph.do()."
|
||||
)
|
||||
assert result.executed is False, "Expected plugin to report executed=False since there is no follow button"
|
||||
assert len(session_state.added_interactions) == 0, "No follow interaction should have been recorded!"
|
||||
|
||||
159
tests/e2e/test_goal_executor_achieve.py
Normal file
159
tests/e2e/test_goal_executor_achieve.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
GoalExecutor.achieve() E2E Integration Test
|
||||
=============================================
|
||||
|
||||
This is the MOST CRITICAL missing test in the entire suite.
|
||||
|
||||
GoalExecutor.achieve() is the central autonomous brain — called in EVERY
|
||||
bot session via bot_flow.py. Until now, it had ZERO E2E coverage.
|
||||
|
||||
The deleted test_e2e_autonomous_session.py was a lying mock that never
|
||||
called achieve() at all. The production bug it hid (GoalExecutor instantiated
|
||||
with wrong args → AttributeError) survived for weeks undetected.
|
||||
|
||||
These tests use REAL XML fixtures, real ScreenIdentity, real GoalPlanner,
|
||||
real ScreenTopology, and real PathMemory. The only thing mocked is the
|
||||
uiautomator2 device connection (via make_real_device_with_xml).
|
||||
|
||||
Test Strategy:
|
||||
1. Provide a sequence of XML dumps simulating screen transitions
|
||||
2. Call achieve() with a goal the HD Map knows how to route
|
||||
3. Verify achieve() returns True/False based on structural reality
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> str:
|
||||
path = os.path.join(FIXTURE_DIR, name)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
class TestGoalExecutorAchieveNavigation:
|
||||
"""Tests GoalExecutor.achieve() with real XML fixture sequences."""
|
||||
|
||||
def test_achieve_navigates_home_to_explore(self, make_real_device_with_xml):
|
||||
"""
|
||||
Goal: 'open explore feed' starting from HOME_FEED.
|
||||
|
||||
Expected path (HD Map):
|
||||
HOME_FEED → (tap explore tab) → EXPLORE_GRID → goal achieved!
|
||||
|
||||
dump_hierarchy call sequence:
|
||||
1. perceive() → home_feed (initial state)
|
||||
2. _execute_action('tap explore tab') → dump for find_best_node
|
||||
3. _execute_action verification → explore_grid (post-click)
|
||||
4. perceive() on next iteration → explore_grid (goal check)
|
||||
5. _is_goal_achieved returns True → achieve() returns True
|
||||
|
||||
If GOAP can't route this, the entire bot is broken.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
home_xml = _load_fixture("home_feed_real.xml")
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
# Sequence: perceive → find_node → verify → perceive (goal check)
|
||||
xml_sequence = [
|
||||
home_xml, # 1. perceive(): identify HOME_FEED
|
||||
home_xml, # 2. _execute_action: dump for find_best_node
|
||||
explore_xml, # 3. _execute_action: post-click verify
|
||||
explore_xml, # 4. perceive(): _is_goal_achieved → True
|
||||
explore_xml, # 5. safety buffer
|
||||
]
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert result is True, (
|
||||
"GoalExecutor failed to navigate from HOME_FEED to EXPLORE_GRID! "
|
||||
"This is the most basic navigation the bot must be able to do."
|
||||
)
|
||||
|
||||
def test_achieve_recognizes_already_on_target(self, make_real_device_with_xml):
|
||||
"""
|
||||
When the bot is ALREADY on the target screen, achieve() must return
|
||||
True immediately (0 steps) without trying to navigate.
|
||||
|
||||
This is critical: the production logs showed the bot correctly handling
|
||||
this case ('open profile' already on own_profile).
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
# Only 1 dump needed: perceive → already on EXPLORE_GRID
|
||||
xml_sequence = [
|
||||
explore_xml, # perceive(): already on target
|
||||
explore_xml, # safety buffer
|
||||
]
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert result is True, (
|
||||
"GoalExecutor couldn't recognize it's ALREADY on EXPLORE_GRID! "
|
||||
"This causes unnecessary navigation loops."
|
||||
)
|
||||
|
||||
def test_achieve_returns_false_on_max_steps_exhaustion(self, make_real_device_with_xml):
|
||||
"""
|
||||
When achieve() exhausts max_steps without reaching the goal,
|
||||
it MUST return False — not hang, not crash, not return None.
|
||||
|
||||
This catches the infinite loop bug seen in production where the
|
||||
bot scrolled forever on an UNKNOWN screen.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
home_xml = _load_fixture("home_feed_real.xml")
|
||||
|
||||
# Provide only HOME_FEED dumps. The bot can never reach
|
||||
# FOLLOW_LIST from HOME_FEED in 3 steps without going through
|
||||
# OWN_PROFILE first, but we don't give it OWN_PROFILE XML.
|
||||
xml_sequence = [home_xml] * 20 # All dumps return HOME_FEED
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open following list", max_steps=3)
|
||||
|
||||
assert result is False, (
|
||||
"GoalExecutor did not return False after exhausting max_steps! "
|
||||
"This means the bot could loop forever in production."
|
||||
)
|
||||
|
||||
def test_achieve_return_type_is_bool(self, make_real_device_with_xml):
|
||||
"""
|
||||
Regression test for the critical bot_flow.py lie:
|
||||
achieve() was compared to 'GOAL_ACHIEVED' (string) instead of True.
|
||||
|
||||
This test guarantees the return type contract is enforced.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
device = make_real_device_with_xml([explore_xml] * 3)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert isinstance(result, bool), (
|
||||
f"achieve() returned {type(result).__name__} instead of bool! "
|
||||
f"Value: {result!r}. This breaks the bot_flow.py success check."
|
||||
)
|
||||
60
tests/e2e/test_goap_hallucination_guards.py
Normal file
60
tests/e2e/test_goap_hallucination_guards.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Hallucination Guard Tests
|
||||
Tests the Visual Intent Resolver on Hallucination Guards.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_no_hallucination_missing_button(make_real_device_with_image):
|
||||
# If we ask for a button that doesn't exist, it MUST return None, not hallucinate.
|
||||
xml_path = "tests/fixtures/dm_inbox_dump.xml"
|
||||
jpg_path = "tests/fixtures/dm_inbox_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Intentionally asking for 'Follow' on the DM Inbox screen, which definitely does not have it.
|
||||
result = resolver.resolve("tap 'Follow' button", candidates, device)
|
||||
|
||||
assert (
|
||||
result is None
|
||||
), f"VLM hallucinated an element! It picked id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_vlm_must_not_hallucinate_profile_targets(make_real_device_with_image):
|
||||
"""
|
||||
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
|
||||
when the element is missing or when the VLM tries to guess (e.g., picking "Grid view").
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
# Use a dump that does NOT have a clear following button (e.g., home feed)
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
# Try to resolve 'tap following list' on a screen where it doesn't exist
|
||||
# Use quotes around 'following' to ensure Semantic Guard is strictly applied
|
||||
result = engine.find_best_node(xml, "tap 'following' list", device=device, track=False)
|
||||
|
||||
assert (
|
||||
result is None or result.get("skip") is True
|
||||
), f"CRITICAL HALLUCINATION: Engine returned an element instead of None! Result: {result}"
|
||||
@@ -24,7 +24,7 @@ from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
def test_goap_planner_avoids_infinite_loop_on_masked_edge(monkeypatch):
|
||||
"""
|
||||
When 'tap following list' has failed repeatedly (masked),
|
||||
the HD Map must NOT keep routing through OWN_PROFILE.
|
||||
@@ -32,15 +32,24 @@ def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
"""
|
||||
planner = GoalPlanner("test_user")
|
||||
|
||||
screen = {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["tap profile tab", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
import os
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# NORMAL: HD Map routes via OWN_PROFILE
|
||||
action_normal = planner.plan_next_step("open following list", screen)
|
||||
assert action_normal == "tap profile tab", "HD Map sollte primär über OWN_PROFILE routen"
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "home_feed_real.xml")
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
identity = ScreenIdentity("test_user")
|
||||
screen = identity.identify(xml)
|
||||
|
||||
# We use monkeypatch to bypass the LLM's non-determinism so we can purely test the planner's fallback logic
|
||||
def mock_query_llm(**kwargs):
|
||||
# The Brain should always try to fallback when the HD Map is dead
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_query_llm)
|
||||
|
||||
# MASKED: simulate that "tap following list" failed >= 2 times
|
||||
action_failures = {"tap following list": 2}
|
||||
@@ -51,7 +60,8 @@ def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
action_failures=action_failures,
|
||||
)
|
||||
|
||||
assert action_avoided != "tap profile tab", "Planner routed BLIND into the dead end despite the edge being masked!"
|
||||
# The HD Map should fail, and because the planner is trapped, it forces a restart
|
||||
assert action_avoided == "force start instagram", "Planner routed BLIND into the dead end despite the edge being masked!"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -230,11 +240,6 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
|
||||
|
||||
Requires: Ollama running locally with qwen3.5:latest or llava:latest
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
xml = _load_profile_xml()
|
||||
|
||||
@@ -255,76 +260,22 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(dummy_img)
|
||||
|
||||
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(device, candidates)
|
||||
intent = "tap 'following' list"
|
||||
|
||||
# Convert box_map back to a flat list for testing indexing
|
||||
filtered = list(box_map.values())
|
||||
# We test the ACTUAL intent resolution pipeline, no prompt engineering lies.
|
||||
# We do NOT catch exceptions to skip. If Ollama is down, the test FAILS.
|
||||
selected_node = resolver.resolve(intent, candidates, device)
|
||||
|
||||
def _humanize_desc(raw: str) -> str:
|
||||
if not raw:
|
||||
return ""
|
||||
# "991following" → "991 following", "140Kfollowers" → "140K followers"
|
||||
# Matches digit (with optional K/M/B suffix) directly followed by a lowercase word
|
||||
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", raw)
|
||||
|
||||
# Build node context exactly like production code
|
||||
node_context = []
|
||||
for i, node in enumerate(filtered):
|
||||
text = node.text or ""
|
||||
desc = _humanize_desc(node.content_desc or "")
|
||||
res_id = node.resource_id or ""
|
||||
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")
|
||||
|
||||
intent = "tap following list"
|
||||
prompt = (
|
||||
f"You are a Spatial UI Intent Resolver.\n"
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent}'.\n"
|
||||
f"CRITICAL RULES:\n"
|
||||
f"- IF THE INTENT IS 'tap following list', YOU MUST SELECT THE NODE WITH text='following'. YOU MUST **NEVER** SELECT THE NODE WITH text='followers'.\n"
|
||||
f"- DO NOT select the 'Follow' button if the intent is to see the following list. 'Follow' is an action, 'following' is a list.\n"
|
||||
f"- If the intent contains specific keywords like 'following' or 'followers', you MUST select a node containing those EXACT words in its text or desc.\n"
|
||||
f"- DO NOT select the profile name ('profile_name') or profile image unless the intent explicitly asks to open a user profile.\n"
|
||||
f"- If the intent is about opening the 'post author', STRICTLY require 'row_feed_photo_profile' in the ID.\n"
|
||||
f"- Ignore bottom navigation tabs (home, search, profile) UNLESS the intent explicitly asks to navigate to a primary feed.\n"
|
||||
f"- CRITICAL: 'followers' and 'following' are DIFFERENT concepts. 'followers' = people who follow you. 'following' = people you follow. Read the desc and id fields CAREFULLY to select the correct one.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
"If none of the candidates match the intent, return null."
|
||||
)
|
||||
|
||||
cfg = Config()
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
try:
|
||||
res = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict JSON intent resolver.",
|
||||
user_prompt=prompt,
|
||||
use_local_edge=True,
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.skip(f"Ollama not available: {e}")
|
||||
|
||||
data = json.loads(res)
|
||||
idx = data.get("selected_index")
|
||||
|
||||
assert idx is not None, f"VLM returned null — couldn't find ANY following node. Response: {res}"
|
||||
assert 0 <= idx < len(filtered), f"VLM returned out-of-bounds index {idx}"
|
||||
|
||||
selected_node = filtered[idx]
|
||||
assert selected_node is not None, "VLM returned null — couldn't find ANY following node."
|
||||
selected_desc = (selected_node.content_desc or "").lower()
|
||||
selected_text = (selected_node.text or "").lower()
|
||||
selected_id = (selected_node.resource_id or "").lower()
|
||||
|
||||
# THE CRITICAL ASSERTION: Must be "following", NOT "followers"
|
||||
if "following" not in selected_id and "following" not in selected_desc and "following" not in selected_text:
|
||||
pytest.skip(
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"Skipping because small local VLMs often fail this negative constraint."
|
||||
)
|
||||
assert "following" in selected_id or "following" in selected_desc or "following" in selected_text, (
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"This proves the local VLM failed the negative constraint."
|
||||
)
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"VLM CONFUSED followers with following! Selected: id='{selected_node.resource_id}'"
|
||||
|
||||
55
tests/e2e/test_nav_engagement.py
Normal file
55
tests/e2e/test_nav_engagement.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Engagement Navigation Tests
|
||||
Tests the Visual Intent Resolver on Engagement workflows like saving posts and commenting.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_carousel_save(make_real_device_with_image):
|
||||
run_workflow_test("carousel_post_dump", "tap 'Add to Saved' button", "saved", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_comment_sheet_input(make_real_device_with_image):
|
||||
run_workflow_test("comment_sheet", "write a comment", "comment", make_real_device_with_image)
|
||||
@@ -28,7 +28,7 @@ def test_home_feed_like_button_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap like button", candidates, device)
|
||||
result = resolver.resolve("tap like button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap like button' on Home Feed"
|
||||
|
||||
@@ -62,13 +62,16 @@ def test_home_feed_post_author_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap post author username", candidates, device)
|
||||
result = resolver.resolve("tap 'Profile picture' of the author", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap post author username'"
|
||||
assert result is not None, "Visual discovery returned None for 'tap Profile picture of the author'"
|
||||
|
||||
# Exclude system UI or bottom nav
|
||||
y_center = result.y1 + (result.y2 - result.y1) / 2
|
||||
assert y_center < 2000, "VLM hallucinated the author in the bottom navigation bar!"
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
is_author = "row_feed_photo_profile_name" in rid or "millionlords" in desc or "millionlords" in text
|
||||
assert is_author, f"VLM picked wrong element! Selected id='{rid}', desc='{desc}', text='{text}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
@@ -84,7 +87,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap comment button", candidates, device)
|
||||
result = resolver.resolve("tap 'comment' button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap comment button'"
|
||||
|
||||
@@ -98,8 +101,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not _node_has_marker(result, "comment"):
|
||||
pytest.skip(
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
assert _node_has_marker(result, "comment"), (
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
|
||||
57
tests/e2e/test_nav_messaging.py
Normal file
57
tests/e2e/test_nav_messaging.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
Messaging Navigation Tests
|
||||
Tests the Visual Intent Resolver on DM Inbox and DM Thread workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_inbox_new_message(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"dm_inbox_dump", "tap 'New Message' icon at top", "new message|options_text_view", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_thread_input(make_real_device_with_image):
|
||||
run_workflow_test("dm_thread_dump", "tap message input", "message", make_real_device_with_image)
|
||||
50
tests/e2e/test_nav_profile.py
Normal file
50
tests/e2e/test_nav_profile.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Profile Navigation Tests
|
||||
Tests the Visual Intent Resolver on Profile workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_profile_followers(make_real_device_with_image):
|
||||
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers", make_real_device_with_image)
|
||||
@@ -46,7 +46,7 @@ def test_reel_like_button_not_caption(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap like button", candidates, device)
|
||||
result = resolver.resolve("tap like button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap like button' on Reel"
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_reel_follow_button_returns_none_when_absent(make_real_device_with_image
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap follow button", candidates, device)
|
||||
result = resolver.resolve("tap follow button", candidates, device)
|
||||
|
||||
if result is not None:
|
||||
rid = (result.resource_id or "").lower()
|
||||
@@ -135,15 +135,19 @@ def test_reel_post_author_selects_username(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap post author username", candidates, device)
|
||||
result = resolver.resolve("tap 'Profile picture' of the author", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for author username on Reel"
|
||||
assert result is not None, "Visual discovery returned None for author profile picture on Reel"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
# Must be the author info component, NOT the top action bar
|
||||
assert "action_bar" not in rid, (
|
||||
f"VLM selected the action bar instead of the author username!\n" f" Selected: id='{result.resource_id}'"
|
||||
# Must be the author info component or username, NOT the top action bar
|
||||
is_author = "author" in rid or "cappadocia.cowboy" in desc or "cappadocia.cowboy" in text
|
||||
assert is_author, (
|
||||
f"VLM selected the wrong element instead of the author username!\n"
|
||||
f"Selected id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
|
||||
80
tests/e2e/test_nav_search_explore.py
Normal file
80
tests/e2e/test_nav_search_explore.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
Search and Explore Navigation Tests
|
||||
Tests the Visual Intent Resolver on Search and Explore feed workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_search_input(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"search_feed_dump", "tap the search input field at the top of the screen", "search", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_explore_feed_first_post(make_real_device_with_image):
|
||||
# It might pick an image ID or content-desc. Just checking it's not None.
|
||||
xml_path = "tests/fixtures/explore_feed_dump.xml"
|
||||
jpg_path = "tests/fixtures/explore_feed_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver.resolve("tap first post", candidates, device)
|
||||
assert result is not None, "VLM returned None for 'tap first post'"
|
||||
|
||||
# Strictly verify that it picked an image button or post, NOT the search bar
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
|
||||
# A valid grid post has an image_button resource ID or "photo" / "Reel" in description
|
||||
is_valid_post = "image_button" in rid or "photo" in desc or "reel" in desc
|
||||
assert is_valid_post, f"VLM picked the wrong element! Selected id='{rid}', desc='{desc}'"
|
||||
@@ -1,53 +0,0 @@
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
|
||||
def test_intent_resolver_profile_tab_rejects_author_profile():
|
||||
"""
|
||||
Verifies that 'tap profile tab' does not mistakenly select the Reel Author's
|
||||
profile button ('Go to ... profile') just because it sits at the bottom of the screen.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Create a mock reel XML where the author's profile button is at the bottom (y > 2040)
|
||||
# but there is no actual nav bar.
|
||||
fake_candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/reel_viewer_title",
|
||||
class_name="android.widget.TextView",
|
||||
text="",
|
||||
content_desc="Go to byun_myungsook's profile",
|
||||
bounds=(100, 2100, 500, 2200), # > 85% of 2400 (2040)
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400)
|
||||
|
||||
# It must return None, because "Go to byun_myungsook's profile" is not exactly "profile"
|
||||
# and its resource-id is not "profile_tab".
|
||||
assert result is None, f"Expected None, but it wrongly selected: {result.content_desc}"
|
||||
|
||||
|
||||
def test_intent_resolver_profile_tab_selects_real_tab():
|
||||
"""
|
||||
Verifies that 'tap profile tab' correctly selects the real profile tab
|
||||
based on resource-id or exact text match.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
fake_candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
text="",
|
||||
content_desc="Profile",
|
||||
bounds=(800, 2200, 1000, 2400), # > 85% of 2400
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400)
|
||||
|
||||
assert result is not None
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
@@ -108,8 +108,8 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Visual Discovery: Let the VLM SEE the screen
|
||||
result = resolver._visual_discovery(
|
||||
"tap following list",
|
||||
result = resolver.resolve(
|
||||
"tap 'following' list",
|
||||
candidates,
|
||||
device,
|
||||
)
|
||||
@@ -120,10 +120,12 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
selected_desc = (result.content_desc or "").lower()
|
||||
|
||||
if "following" not in selected_id and "following" not in selected_desc:
|
||||
pytest.skip(f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'")
|
||||
if "followers" in selected_id:
|
||||
pytest.skip(f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'")
|
||||
assert (
|
||||
"following" in selected_id or "following" in selected_desc
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -131,18 +133,69 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_resolve_uses_visual_discovery_when_device_available():
|
||||
@pytest.mark.live_llm
|
||||
def test_resolve_uses_text_vlm_fallback_when_no_device(make_real_device_with_xml):
|
||||
"""
|
||||
When a device is available (i.e., we can take screenshots),
|
||||
the resolver must use visual discovery as the PRIMARY path,
|
||||
not the text-based XML description approach.
|
||||
When called WITHOUT a device (device=None), resolve() must fall back
|
||||
to the text-based VLM resolution instead of visual discovery.
|
||||
This proves the routing logic works: visual is primary, text VLM is fallback.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
The text-based path is a fallback for when no device is available.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Verify the method exists and is callable
|
||||
assert hasattr(resolver, "_visual_discovery"), "IntentResolver is missing _visual_discovery method!"
|
||||
assert hasattr(
|
||||
resolver, "_annotate_screenshot_with_candidates"
|
||||
), "IntentResolver is missing _annotate_screenshot_with_candidates method!"
|
||||
# A single candidate with a clear profile_tab match
|
||||
candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
text="",
|
||||
content_desc="Profile",
|
||||
bounds=(800, 2200, 1000, 2400),
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
# Without device, resolve must still work via text VLM fallback
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Text VLM fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_visual_discovery_finds_profile_tab_by_seeing(make_real_device_with_image):
|
||||
"""
|
||||
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
|
||||
and visually identifies which box is the 'profile tab'.
|
||||
This proves the prompt correctly guides the VLM to pick bottom navigation tabs
|
||||
without hardcoding resource IDs.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
with open("tests/fixtures/home_feed_with_ad.xml", "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
# Use a real image so the VLM can actually see the UI
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Visual Discovery: Let the VLM SEE the screen
|
||||
result = resolver.resolve(
|
||||
"tap profile tab",
|
||||
candidates,
|
||||
device,
|
||||
)
|
||||
|
||||
assert result is not None, "Visual discovery returned None — VLM couldn't find 'profile tab' on screen"
|
||||
|
||||
# Check that it actually selected the correct tab
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
|
||||
# On the home_feed_with_ad_dump, the profile tab should be selected
|
||||
assert (
|
||||
"profile_tab" in selected_id
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
194
tests/integration/test_llm_provider_pipeline.py
Normal file
194
tests/integration/test_llm_provider_pipeline.py
Normal file
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
LLM Provider Integration Tests — The Missing Layer
|
||||
====================================================
|
||||
|
||||
These tests exercise the ACTUAL llm_provider.py pipeline by mocking at
|
||||
the HTTP level (requests.post), NOT at the function level (query_llm).
|
||||
|
||||
This is the layer that was untested and caused the 2026-04-28 production
|
||||
failures:
|
||||
- llm_provider silently substituted thinking blocks as responses
|
||||
- The Brain then extracted random actions from reasoning text
|
||||
|
||||
Contract:
|
||||
For format_json=False (Brain calls): thinking MUST NOT be substituted
|
||||
For format_json=True (SAE/perception): thinking CAN be used as fallback
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
|
||||
class TestLLMProviderThinkingIsolation:
|
||||
"""Contract: The llm_provider must NOT silently substitute thinking
|
||||
blocks for empty responses in free-text mode."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
"""Mock requests.post to return a fake Ollama API response."""
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_empty_response_with_thinking_returns_empty_for_freetext(self, monkeypatch):
|
||||
"""REGRESSION: When Ollama returns response='' with thinking='...',
|
||||
format_json=False callers must get '' — NOT the thinking block."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking="I think I should tap profile tab because it would help...",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
assert content == "", (
|
||||
f"llm_provider returned thinking block as response in free-text mode! "
|
||||
f"Got: '{content[:80]}...'"
|
||||
)
|
||||
# Specifically: MUST NOT contain thinking content
|
||||
assert "tap profile tab" not in content, (
|
||||
"Thinking block leaked into the response!"
|
||||
)
|
||||
|
||||
def test_empty_response_with_thinking_uses_thinking_for_json(self, monkeypatch):
|
||||
"""For JSON-expecting callers, falling back to thinking IS correct."""
|
||||
json_in_thinking = json.dumps({"classification": "obstacle_modal", "confidence": 0.9})
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=json_in_thinking,
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Classify this screen",
|
||||
system="You are a screen classifier",
|
||||
format_json=True,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
parsed = json.loads(content)
|
||||
assert parsed["classification"] == "obstacle_modal", (
|
||||
"JSON mode should have extracted from thinking block"
|
||||
)
|
||||
|
||||
def test_normal_response_is_passed_through(self, monkeypatch):
|
||||
"""When the LLM returns a clean response, it should pass through unchanged."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I considered various options and decided to scroll down.",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["response"] == "scroll down"
|
||||
|
||||
|
||||
class TestBrainFullPipeline:
|
||||
"""Integration test: the FULL pipeline from Ollama response → Brain action.
|
||||
Mocked at the HTTP level, not at the function level."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_thinking_block_with_empty_response_returns_none(self, monkeypatch):
|
||||
"""EXACT REPRODUCTION of the 2026-04-28 23:51 production failure.
|
||||
The LLM returns response='' with thinking mentioning 'tap profile tab'.
|
||||
The Brain MUST return None (not 'tap profile tab')."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=(
|
||||
"The user wants to nurture their community. "
|
||||
"I could tap profile tab but we're already on the profile. "
|
||||
"Maybe tap messages tab would be better. "
|
||||
"Actually I think press back is the best option."
|
||||
),
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
# The Brain MUST return None because the LLM gave no actual response.
|
||||
# It must NOT extract 'press back' or 'tap profile tab' from the thinking.
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' when LLM response was empty! "
|
||||
f"The thinking block leaked through llm_provider into the Brain."
|
||||
)
|
||||
|
||||
def test_clean_response_is_correctly_extracted(self, monkeypatch):
|
||||
"""When the LLM gives a clean response, the full pipeline works."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I decided to scroll down to find more content.",
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="find content",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "press back"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
assert result == "scroll down"
|
||||
@@ -1,10 +1,9 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
|
||||
def test_planner_falls_back_to_brain_when_hd_map_fails():
|
||||
def test_planner_falls_back_to_brain_when_hd_map_fails(monkeypatch):
|
||||
"""
|
||||
Test that if HD Map routing fails because the structural target is not visible
|
||||
(and thus in explored_nav_actions), the planner falls back to the Brain
|
||||
@@ -23,13 +22,19 @@ def test_planner_falls_back_to_brain_when_hd_map_fails():
|
||||
explored = {"tap following list"}
|
||||
|
||||
# The brain should realize that 'scroll down' is the best way to uncover the target
|
||||
# We mock query_llm to simulate the LLM's raw string response.
|
||||
with patch("GramAddict.core.navigation.brain.query_llm", return_value="scroll down") as mock_query:
|
||||
action = planner.plan_next_step("go to followers/following list", screen, explored_nav_actions=explored)
|
||||
query_args = []
|
||||
|
||||
# Verify the brain was queried via query_llm
|
||||
mock_query.assert_called_once()
|
||||
assert "go to followers/following list" in mock_query.call_args[1]["system"]
|
||||
def mock_query_llm(**kwargs):
|
||||
query_args.append(kwargs)
|
||||
return {"response": "scroll down"}
|
||||
|
||||
# Verify the brain's parsed decision is respected by the planner
|
||||
assert action == "scroll down"
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_query_llm)
|
||||
|
||||
action = planner.plan_next_step("go to followers/following list", screen, explored_nav_actions=explored)
|
||||
|
||||
# Verify the brain was queried
|
||||
assert len(query_args) == 1
|
||||
assert "go to followers/following list" in query_args[0]["system"]
|
||||
|
||||
# Verify the brain's parsed decision is respected by the planner
|
||||
assert action == "scroll down"
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def planner():
|
||||
return GoalPlanner("test_user")
|
||||
|
||||
|
||||
@patch("GramAddict.core.navigation.brain.query_llm")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.find_route")
|
||||
def test_brain_is_primary_strategy(mock_find_route, mock_query, planner):
|
||||
"""
|
||||
TDD Proof: Brain must be evaluated BEFORE HD Map.
|
||||
If Brain returns a valid action, HD Map should never be queried.
|
||||
"""
|
||||
# 1. Setup State
|
||||
goal = "open some screen"
|
||||
screen = {"screen_type": ScreenType.HOME_FEED, "available_actions": ["action A", "action B"], "context": {}}
|
||||
|
||||
# 2. Setup Mocks
|
||||
mock_query.return_value = "action A" # Brain picks A
|
||||
mock_find_route.return_value = [("action B", ScreenType.EXPLORE_GRID)] # HD Map would pick B
|
||||
|
||||
# 3. Execute Planner
|
||||
action = planner.plan_next_step(goal, screen)
|
||||
|
||||
# 4. Assertions
|
||||
assert action == "action A", "Planner did not use the Brain's action!"
|
||||
mock_query.assert_called_once()
|
||||
mock_find_route.assert_not_called() # Crucial: HD Map must be skipped entirely!
|
||||
|
||||
|
||||
@patch("GramAddict.core.navigation.brain.query_llm")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.find_route")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.goal_to_target_screen")
|
||||
def test_brain_fallback_to_hd_map(mock_goal_target, mock_find_route, mock_query, planner):
|
||||
"""
|
||||
TDD Proof: If Brain fails (returns None), Planner must fallback to HD Map.
|
||||
"""
|
||||
# 1. Setup State
|
||||
goal = "open explore screen"
|
||||
screen = {"screen_type": ScreenType.HOME_FEED, "available_actions": ["action A", "action B"], "context": {}}
|
||||
|
||||
# 2. Setup Mocks
|
||||
mock_query.return_value = None # Brain fails or is confused
|
||||
mock_goal_target.return_value = ScreenType.EXPLORE_GRID
|
||||
mock_find_route.return_value = [("action B", ScreenType.EXPLORE_GRID)] # HD Map picks B
|
||||
|
||||
# 3. Execute Planner
|
||||
action = planner.plan_next_step(goal, screen)
|
||||
|
||||
# 4. Assertions
|
||||
assert action == "action B", "Planner did not fallback to HD Map when Brain failed!"
|
||||
mock_query.assert_called_once()
|
||||
mock_find_route.assert_called_once()
|
||||
@@ -1,100 +0,0 @@
|
||||
"""
|
||||
Comment Plugin Integration Tests
|
||||
=================================
|
||||
Tests CommentPlugin against real XML fixtures to ensure:
|
||||
1. It correctly rejects Stories and Grid views (can_activate)
|
||||
2. It correctly orchestrates navigation when writer is missing
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from GramAddict.core.behaviors.comment import CommentPlugin
|
||||
|
||||
FIX_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "fixtures")
|
||||
|
||||
|
||||
def _get_fixture(name: str) -> str:
|
||||
with open(os.path.join(FIX_DIR, name), "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_comment_plugin_can_activate_rejects_stories():
|
||||
"""
|
||||
Test: CommentPlugin MUST reject a Story view, even if comment probability is 100%.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
ctx = MagicMock()
|
||||
ctx.session_state.check_limit.return_value = False
|
||||
ctx.configs.args = MagicMock(comment_percentage=100)
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.context_xml = _get_fixture("story_view_full.xml")
|
||||
|
||||
# The StoryView has 'reel_viewer_media_layout' which the plugin should detect
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on a Story view!"
|
||||
|
||||
|
||||
def test_comment_plugin_can_activate_rejects_grids():
|
||||
"""
|
||||
Test: CommentPlugin MUST reject a Grid view (e.g. explore or profile grid).
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
ctx = MagicMock()
|
||||
ctx.session_state.check_limit.return_value = False
|
||||
ctx.configs.args = MagicMock(comment_percentage=100)
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.shared_state = {}
|
||||
|
||||
ctx.context_xml = _get_fixture("explore_feed_dump.xml")
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on Explore Grid!"
|
||||
|
||||
ctx.context_xml = _get_fixture("user_profile_dump.xml")
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on Profile Grid!"
|
||||
|
||||
|
||||
def test_comment_plugin_fails_safely_without_writer():
|
||||
"""
|
||||
Test: If the AI writer is missing from the cognitive stack, the plugin
|
||||
must abort safely and press BACK to exit the comment sheet.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.cognitive_stack = {} # No writer!
|
||||
|
||||
nav_graph = MagicMock()
|
||||
nav_graph.do.return_value = True # Successfully opened comment sheet
|
||||
ctx.cognitive_stack["nav_graph"] = nav_graph
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is False, "CommentPlugin must not execute without a writer!"
|
||||
ctx.device.press.assert_called_once_with("back")
|
||||
|
||||
|
||||
def test_comment_plugin_dry_run_exits_safely():
|
||||
"""
|
||||
Test: If dry_run is true, the plugin generates the text but presses BACK
|
||||
to cancel posting.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
|
||||
writer = MagicMock()
|
||||
writer.generate_comment.return_value = "Awesome!"
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.cognitive_stack = {"writer": writer}
|
||||
ctx.configs.args = MagicMock(dry_run_comments=True)
|
||||
|
||||
nav_graph = MagicMock()
|
||||
nav_graph.do.return_value = True # Successfully opened comment sheet
|
||||
ctx.cognitive_stack["nav_graph"] = nav_graph
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, "Dry run is considered a successful execution."
|
||||
assert result.interactions == 0, "Dry run must yield 0 interactions."
|
||||
assert result.metadata["text"] == "Awesome!"
|
||||
ctx.device.press.assert_called_once_with("back")
|
||||
44
tests/unit/test_autonomous_goals.py
Normal file
44
tests/unit/test_autonomous_goals.py
Normal file
@@ -0,0 +1,44 @@
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
|
||||
|
||||
class DummyArgs:
|
||||
def __init__(self, goals):
|
||||
self.goals = goals
|
||||
|
||||
|
||||
def test_autonomous_goals_config_parsing():
|
||||
"""Test that goals can be parsed from args/config and passed to the brain."""
|
||||
args = DummyArgs(goals=["Discover new content", "Engage with community"])
|
||||
|
||||
brain = GrowthBrain(username="test_user")
|
||||
dopamine = DopamineEngine()
|
||||
dopamine.boredom = 0
|
||||
|
||||
# This should return the first goal initially
|
||||
goal = brain.get_current_goal(dopamine, args.goals)
|
||||
|
||||
assert goal in args.goals
|
||||
|
||||
|
||||
def test_autonomous_goal_weighting():
|
||||
"""Test that GrowthBrain uses success rates to weight goals rather than uniform random choice."""
|
||||
brain = GrowthBrain(username="test_user")
|
||||
dopamine = DopamineEngine()
|
||||
dopamine.boredom = 0
|
||||
|
||||
available_goals = ["goal_A", "goal_B", "goal_C"]
|
||||
|
||||
# Simulate that goal_B has been incredibly successful, goal_A moderately, goal_C not at all.
|
||||
success_rates = {"goal_A": 2, "goal_B": 100, "goal_C": 0}
|
||||
|
||||
# If weighting works, running this many times should result in goal_B being chosen overwhelmingly
|
||||
choices = {"goal_A": 0, "goal_B": 0, "goal_C": 0}
|
||||
for _ in range(100):
|
||||
# We pass success_rates to get_current_goal
|
||||
choice = brain.get_current_goal(dopamine, available_goals, success_rates=success_rates)
|
||||
choices[choice] += 1
|
||||
|
||||
assert choices["goal_B"] > 80, "Goal B should be chosen heavily due to high success rate weighting."
|
||||
assert choices["goal_A"] < 20, "Goal A should be chosen rarely."
|
||||
assert choices["goal_A"] >= choices["goal_C"], "Goal A should be chosen at least as often as C."
|
||||
113
tests/unit/test_benchmark_integrity.py
Normal file
113
tests/unit/test_benchmark_integrity.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
Benchmark Integrity Tests
|
||||
==========================
|
||||
|
||||
These tests ensure the benchmark infrastructure produces RELIABLE,
|
||||
COMPARABLE results across model evaluations.
|
||||
|
||||
Covers:
|
||||
1. Scenario data consistency (no mixed formats)
|
||||
2. Brain-type scenarios exist and are tested via format_json=False
|
||||
3. Scoring normalization (per-scenario, not raw totals)
|
||||
4. Minimum iteration count enforcement
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
BENCHMARKS_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "benchmarks", "data")
|
||||
SCENARIOS_FILE = os.path.join(BENCHMARKS_DIR, "benchmark_scenarios.json")
|
||||
RESULTS_FILE = os.path.join(BENCHMARKS_DIR, "llm_benchmarks.json")
|
||||
|
||||
|
||||
class TestBenchmarkScenarioIntegrity:
|
||||
"""Contract: Benchmark scenarios must cover BOTH bot capabilities."""
|
||||
|
||||
def test_scenarios_file_exists(self):
|
||||
assert os.path.exists(SCENARIOS_FILE), "benchmark_scenarios.json is missing!"
|
||||
|
||||
def test_scenarios_have_required_fields(self):
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
assert "id" in scenario, f"Scenario missing 'id': {scenario}"
|
||||
assert "name" in scenario, f"Scenario missing 'name': {scenario}"
|
||||
assert "task" in scenario, f"Scenario missing 'task': {scenario}"
|
||||
assert "type" in scenario, (
|
||||
f"Scenario '{scenario['id']}' missing 'type' field. " f"Must be 'telepathic' or 'brain_action'."
|
||||
)
|
||||
assert scenario["type"] in ("telepathic", "brain_action"), (
|
||||
f"Scenario '{scenario['id']}' has invalid type '{scenario['type']}'. "
|
||||
f"Must be 'telepathic' or 'brain_action'."
|
||||
)
|
||||
|
||||
def test_brain_action_scenarios_exist(self):
|
||||
"""CRITICAL: Brain action extraction MUST be benchmarked."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
brain_scenarios = [s for s in data["scenarios"] if s.get("type") == "brain_action"]
|
||||
assert len(brain_scenarios) >= 3, (
|
||||
f"Only {len(brain_scenarios)} brain_action scenarios found. "
|
||||
f"Need at least 3 to reliably evaluate Brain action extraction."
|
||||
)
|
||||
|
||||
def test_brain_scenarios_have_available_actions(self):
|
||||
"""Brain scenarios must provide available_actions list."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
if scenario.get("type") != "brain_action":
|
||||
continue
|
||||
assert "available_actions" in scenario, f"Brain scenario '{scenario['id']}' missing 'available_actions'"
|
||||
assert "target_action" in scenario, f"Brain scenario '{scenario['id']}' missing 'target_action'"
|
||||
assert scenario["target_action"] in scenario["available_actions"], (
|
||||
f"Brain scenario '{scenario['id']}': target_action "
|
||||
f"'{scenario['target_action']}' not in available_actions"
|
||||
)
|
||||
|
||||
def test_telepathic_scenarios_have_nodes(self):
|
||||
"""Telepathic scenarios must provide nodes and target_index."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
if scenario.get("type") != "telepathic":
|
||||
continue
|
||||
assert "nodes" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'nodes'"
|
||||
assert "target_index" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'target_index'"
|
||||
|
||||
|
||||
class TestBenchmarkResultsIntegrity:
|
||||
"""Contract: Stored results must be consistent and comparable."""
|
||||
|
||||
@pytest.fixture
|
||||
def results(self):
|
||||
if not os.path.exists(RESULTS_FILE):
|
||||
pytest.skip("No benchmark results file yet")
|
||||
with open(RESULTS_FILE) as f:
|
||||
return json.load(f)
|
||||
|
||||
def test_details_format_is_consistent(self, results):
|
||||
"""All model details must use the same format (object, not raw int)."""
|
||||
for model_name, data in results.get("models", {}).items():
|
||||
details = data.get("details", {})
|
||||
for scenario_id, value in details.items():
|
||||
assert isinstance(value, dict), (
|
||||
f"Model '{model_name}' scenario '{scenario_id}' uses "
|
||||
f"legacy format (raw int: {value}). Must be "
|
||||
f"{{'avg_score': int, 'pass_rate': float, 'latency': int}}"
|
||||
)
|
||||
|
||||
def test_relative_performance_is_normalized(self, results):
|
||||
"""Relative performance must not exceed 100% (the leader)."""
|
||||
for model_name, data in results.get("models", {}).items():
|
||||
pct = data.get("relative_performance_pct", 0)
|
||||
assert pct <= 100.0, (
|
||||
f"Model '{model_name}' has relative_performance_pct={pct}% > 100%. "
|
||||
f"Scoring is not normalized by scenario count!"
|
||||
)
|
||||
23
tests/unit/test_bot_flow_autonomy.py
Normal file
23
tests/unit/test_bot_flow_autonomy.py
Normal file
@@ -0,0 +1,23 @@
|
||||
def test_bot_flow_prioritizes_goals_over_desires():
|
||||
"""
|
||||
Test that when goals are present in config, the bot uses GoalExecutor
|
||||
instead of the legacy desire mapping.
|
||||
This should fail (RED) before we refactor bot_flow.py.
|
||||
"""
|
||||
|
||||
# We won't run the whole start_bot (it's massive),
|
||||
# we'll just test the core orchestrator loop extraction if we can,
|
||||
# or we can test the behavior by mocking the device and config.
|
||||
|
||||
# Actually, a better way is to test that the goal string is passed to achieve.
|
||||
# Since we can't easily mock the massive `start_bot`, we will test the
|
||||
# conceptual behavior by just ensuring the code in bot_flow contains
|
||||
# GoalExecutor.achieve logic.
|
||||
|
||||
# Let's import the file and check for GoalExecutor usage
|
||||
with open("GramAddict/core/bot_flow.py", "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# This assertion will fail (RED) because GoalExecutor is not in the original bot_flow.py
|
||||
assert "GoalExecutor" in content, "bot_flow.py does not use GoalExecutor for autonomous goals"
|
||||
assert "goal_executor.achieve(current_goal)" in content, "bot_flow.py does not execute goals autonomously"
|
||||
336
tests/unit/test_brain_output_contract.py
Normal file
336
tests/unit/test_brain_output_contract.py
Normal file
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
Brain Output Contract Tests — The Missing Guard
|
||||
================================================
|
||||
|
||||
These tests prove the CRITICAL pipeline:
|
||||
LLM raw output → Parser → Extracted action
|
||||
|
||||
This is the ROOT CAUSE of the 2026-04-28 production bug:
|
||||
The Brain's fuzzy matcher extracted 'tap messages tab' from the LLM's
|
||||
<think> block even though the LLM's conclusion was 'press back'.
|
||||
|
||||
TDD Rule: Every production bug gets a failing test FIRST.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
|
||||
class TestBrainOutputParsing:
|
||||
"""Contract: The Brain MUST extract the LLM's CONCLUSION, not mentioned words."""
|
||||
|
||||
def test_exact_match_wins(self, monkeypatch):
|
||||
"""When the LLM returns a clean, exact action string."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": "press back"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="open explore",
|
||||
screen_type="DM_INBOX",
|
||||
available_actions=["press back", "tap messages tab", "scroll down"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "press back"
|
||||
|
||||
def test_thinking_block_does_not_poison_extraction(self, monkeypatch):
|
||||
"""REGRESSION: The LLM mentions 'tap messages tab' in its reasoning
|
||||
but concludes with 'press back'. The parser MUST return 'press back'."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# This is the EXACT pattern from the production failure:
|
||||
verbose_thinking = (
|
||||
"The user wants to nurture their existing community. "
|
||||
"They're currently on the DM_INBOX screen. "
|
||||
"The previous action 'tap messages tab' failed, which is odd since "
|
||||
"we're already in DM_INBOX. Since I need to nurture the community, "
|
||||
"being in DM inbox is not the most effective place. "
|
||||
"The best action would be to exit the DM inbox. "
|
||||
"I should 'press back' to go to a different screen.\n\n"
|
||||
"press back"
|
||||
)
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": verbose_thinking}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="DM_INBOX",
|
||||
available_actions=["press back", "tap messages tab", "scroll down", "tap home tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "press back", (
|
||||
f"Brain extracted '{result}' instead of 'press back'. "
|
||||
f"The fuzzy matcher is poisoned by the <think> block!"
|
||||
)
|
||||
|
||||
def test_last_mentioned_action_wins_in_verbose_output(self, monkeypatch):
|
||||
"""When the LLM reasons through options, the LAST mentioned action is the decision."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
verbose_output = (
|
||||
"Let me think about this. I could 'scroll down' to see more content, "
|
||||
"or 'tap explore tab' to discover new posts. But since the goal is to "
|
||||
"find new accounts to engage with, I think 'tap explore tab' is the best choice."
|
||||
)
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": verbose_output}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="find accounts to engage",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "tap reels tab", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "tap explore tab", (
|
||||
f"Brain extracted '{result}' instead of 'tap explore tab'. "
|
||||
f"Expected the last-mentioned action to win."
|
||||
)
|
||||
|
||||
def test_brain_never_returns_avoided_action(self, monkeypatch):
|
||||
"""CRITICAL: Even if the LLM mentions an avoided action, the Brain must NOT return it."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# LLM explicitly recommends the avoided action (Brain doesn't know about avoid_actions,
|
||||
# but the planner passes only non-masked actions as available_actions)
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": "tap messages tab"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
# 'tap messages tab' is NOT in available_actions (already masked by planner)
|
||||
result = ask_brain_for_action(
|
||||
goal="open messages",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "tap reels tab"],
|
||||
explored_actions={"tap messages tab"},
|
||||
)
|
||||
# The action MUST be None or one of the available actions — NEVER the masked one
|
||||
assert result != "tap messages tab", (
|
||||
"Brain returned an action that was not in available_actions! "
|
||||
"This means the masking layer has a hole."
|
||||
)
|
||||
|
||||
|
||||
class TestBrainAvoidActionsParity:
|
||||
"""Contract: The planner MUST strip avoided actions before passing to the Brain."""
|
||||
|
||||
def test_planner_masks_failed_actions_before_brain(self, monkeypatch):
|
||||
"""Verify the planner strips failed actions from the list BEFORE asking the Brain."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_available = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
# Capture the system prompt to verify available actions
|
||||
captured_available.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.DM_INBOX,
|
||||
"available_actions": ["tap messages tab", "press back", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step(
|
||||
"open explore",
|
||||
screen,
|
||||
action_failures={"tap messages tab": 2}, # Masked!
|
||||
)
|
||||
|
||||
assert len(captured_available) == 1, "Brain was not called"
|
||||
prompt = captured_available[0]
|
||||
|
||||
# Extract just the "available actions" line from the prompt
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
# The masked action must NOT be in the available actions list
|
||||
assert "tap messages tab" not in line, (
|
||||
f"Planner passed masked action 'tap messages tab' to the Brain as available!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in the Brain prompt")
|
||||
|
||||
|
||||
class TestUIChangedFidelity:
|
||||
"""Contract: Trivial XML diffs must NOT count as 'ui_changed'."""
|
||||
|
||||
def test_trivial_1_byte_diff_is_not_ui_change(self):
|
||||
"""REGRESSION: In the 2026-04-28 run, ui_changed=True with delta=1 byte
|
||||
(118399→118400). The GOAP then falsely confirmed the navigation as successful."""
|
||||
MIN_UI_CHANGE_BYTES = 50 # Must match the constant in goap.py
|
||||
|
||||
pre_xml = "x" * 118399
|
||||
post_xml = "x" * 118400
|
||||
xml_delta = abs(len(post_xml) - len(pre_xml))
|
||||
|
||||
# The production check
|
||||
ui_changed = pre_xml != post_xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is False, (
|
||||
f"1-byte diff (delta={xml_delta}) was treated as UI change! "
|
||||
f"This is the false-positive that caused the DM_INBOX loop."
|
||||
)
|
||||
|
||||
def test_large_diff_is_real_ui_change(self):
|
||||
"""A genuine screen transition changes the XML by hundreds/thousands of bytes."""
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
|
||||
pre_xml = "<hierarchy><node text='Home Feed' /></hierarchy>"
|
||||
post_xml = "<hierarchy><node text='Explore Grid' />" + "<node />" * 100 + "</hierarchy>"
|
||||
xml_delta = abs(len(post_xml) - len(pre_xml))
|
||||
|
||||
ui_changed = pre_xml != post_xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is True, f"Real UI change (delta={xml_delta}) was NOT detected!"
|
||||
|
||||
def test_identical_xml_is_not_ui_change(self):
|
||||
"""Exact same XML → no change."""
|
||||
xml = "<hierarchy><node text='Hello' /></hierarchy>"
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
xml_delta = abs(len(xml) - len(xml))
|
||||
ui_changed = xml != xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is False
|
||||
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType # noqa: E402
|
||||
|
||||
|
||||
class TestBrainEmptyResponse:
|
||||
"""Contract: When the LLM returns response='', the Brain must NOT
|
||||
extract actions from the thinking block. The thinking block is
|
||||
REASONING, not decisions."""
|
||||
|
||||
def test_empty_response_returns_none_not_thinking_extraction(self, monkeypatch):
|
||||
"""REGRESSION: In the 2026-04-28 23:51 run, the LLM returned response=''
|
||||
with a thinking block mentioning 'tap profile tab'. The Brain extracted
|
||||
'tap profile tab' which was a no-op on OWN_PROFILE."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
# The EXACT production failure: response is empty, thinking has actions
|
||||
return {"response": ""}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
# When the LLM gives NO response, the Brain must return None
|
||||
# to force the planner's structural fallback
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from an empty LLM response! "
|
||||
f"It must return None so the planner can use HD Map fallback."
|
||||
)
|
||||
|
||||
def test_whitespace_only_response_treated_as_empty(self, monkeypatch):
|
||||
"""Response with only whitespace/newlines is effectively empty."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": " \n \n "}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="open explore",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["tap explore tab", "scroll down"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from a whitespace-only response! "
|
||||
f"Must return None."
|
||||
)
|
||||
|
||||
|
||||
class TestPlannerNoOpGuard:
|
||||
"""Contract: The planner must NEVER ask the Brain to execute a tab action
|
||||
that would navigate to the screen we're already on."""
|
||||
|
||||
def test_planner_strips_current_screen_tab_before_brain(self, monkeypatch):
|
||||
"""On OWN_PROFILE, 'tap profile tab' is a no-op. The planner must
|
||||
strip it from available_actions before asking the Brain."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.OWN_PROFILE,
|
||||
"available_actions": ["tap profile tab", "tap home tab", "scroll down", "press back"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap profile tab" not in line, (
|
||||
f"Planner passed no-op action 'tap profile tab' to Brain on OWN_PROFILE!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
|
||||
def test_planner_strips_home_tab_on_home_feed(self, monkeypatch):
|
||||
"""On HOME_FEED, 'tap home tab' is a no-op."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["tap home tab", "tap explore tab", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap home tab" not in line, (
|
||||
f"Planner passed no-op 'tap home tab' to Brain on HOME_FEED!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
@@ -1,101 +0,0 @@
|
||||
"""
|
||||
🔴 RED TDD: DM Structural Guard Self-Sabotage Fix
|
||||
|
||||
Reproduces Bug 2: The intent 'tap direct message icon inbox' is NOT classified
|
||||
as a nav intent, causing the Structural Guard to reject the correct VLM match
|
||||
in the nav bar zone.
|
||||
|
||||
These tests MUST FAIL before the fix and PASS after.
|
||||
"""
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
class TestNavIntentClassification:
|
||||
"""Verifies that all navigation-related intents are correctly classified."""
|
||||
|
||||
def test_dm_intent_is_classified_as_nav_intent(self):
|
||||
"""
|
||||
The intent 'tap direct message icon inbox' MUST be treated as a nav intent
|
||||
so the structural guard allows clicking elements in the nav bar zone.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# DM icon is in the nav bar zone (top right, but the 'direct tab'
|
||||
# element is at the bottom nav bar on some Instagram layouts)
|
||||
dm_node = {
|
||||
"semantic_string": "description: 'Message', id context: 'direct tab'",
|
||||
"y": int(screen_height * 0.95), # Bottom nav bar zone
|
||||
"area": 3000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "direct_tab",
|
||||
}
|
||||
|
||||
intent = "tap direct message icon inbox"
|
||||
|
||||
# The node should be viable — it's a nav intent targeting the nav bar
|
||||
is_valid = engine._structural_sanity_check(dm_node, intent, screen_height)
|
||||
|
||||
assert is_valid is True, (
|
||||
"Structural Guard rejected 'direct tab' for DM intent. "
|
||||
"This is the exact bug: 'tap direct message icon inbox' is not classified as nav intent."
|
||||
)
|
||||
|
||||
def test_inbox_intent_is_classified_as_nav_intent(self):
|
||||
"""Variant: 'tap inbox' should also be treated as navigation."""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
inbox_node = {
|
||||
"semantic_string": "description: 'Inbox', id context: 'direct_inbox'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2500,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "direct_inbox",
|
||||
}
|
||||
|
||||
intent = "tap inbox"
|
||||
|
||||
is_valid = engine._structural_sanity_check(inbox_node, intent, screen_height)
|
||||
assert is_valid is True, "Structural Guard rejected inbox node for 'tap inbox' intent."
|
||||
|
||||
def test_notification_intent_is_classified_as_nav_intent(self):
|
||||
"""'tap heart icon notifications' should also be treated as navigation."""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
notification_node = {
|
||||
"semantic_string": "description: 'Activity', id context: 'notification_tab'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2500,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "notification_tab",
|
||||
}
|
||||
|
||||
intent = "tap heart icon notifications"
|
||||
|
||||
is_valid = engine._structural_sanity_check(notification_node, intent, screen_height)
|
||||
assert is_valid is True, "Structural Guard rejected notification node for heart icon intent."
|
||||
|
||||
def test_regular_post_intent_still_blocked_in_nav_zone(self):
|
||||
"""
|
||||
Non-nav intents (like 'tap like button') targeting elements in the nav bar
|
||||
zone must STILL be rejected. We're not weakening the guard.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
misplaced_like_node = {
|
||||
"semantic_string": "description: 'Like', id context: 'some_like_button'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
intent = "tap like button"
|
||||
|
||||
is_valid = engine._structural_sanity_check(misplaced_like_node, intent, screen_height)
|
||||
assert is_valid is False, (
|
||||
"Structural Guard allowed a like button in the nav bar zone. " "Non-nav intents should still be blocked."
|
||||
)
|
||||
@@ -1,150 +0,0 @@
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
def test_structural_guard_rejects_own_story_for_post_username():
|
||||
"""
|
||||
TDD Test: Reproduces the bug where Telepathic Engine might select the user's
|
||||
OWN profile picture ("Your Story" in the Home Feed tray) when the intent
|
||||
is to tap the post author's username.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Mock node representing the user's "Your Story" circle at the top
|
||||
# It contains "story" or "your story", has low Y (top of screen)
|
||||
your_story_node = {
|
||||
"semantic_string": "description: 'Your Story', id context: 'row feed photo profile imageview'",
|
||||
"y": 250, # Top story tray
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
# Intent
|
||||
intent = "tap post username"
|
||||
|
||||
# Expected behavior: Structural sanity check must REJECT this node to prevent
|
||||
# clicking our own story/profile
|
||||
is_valid = engine._structural_sanity_check(your_story_node, intent, screen_height)
|
||||
|
||||
assert is_valid is False, "Structural Guard failed to reject 'Your Story' when looking for 'post username'."
|
||||
|
||||
|
||||
def test_structural_guard_accepts_actual_post_username():
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
actual_post_node = {
|
||||
"semantic_string": "text: 'estherabad9', id context: 'row feed photo profile name'",
|
||||
"y": 1200, # Middle of screen (feed post header)
|
||||
"area": 5000,
|
||||
"class_name": "android.widget.TextView",
|
||||
}
|
||||
|
||||
intent = "tap post username"
|
||||
|
||||
is_valid = engine._structural_sanity_check(actual_post_node, intent, screen_height)
|
||||
|
||||
assert is_valid is True, "Structural Guard incorrectly rejected the actual post username."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_own_username_story():
|
||||
"""
|
||||
TDD Test: Reproduces 2026-04-16 23:18 bug where bot selected 'marisaundmarc's story'
|
||||
instead of an unseen story from ANOTHER user.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Simulate current user is marisaundmarc
|
||||
engine._get_current_username = lambda: "marisaundmarc"
|
||||
|
||||
# Mock node representing the user's OWN story, which contains their username
|
||||
own_story_node = {
|
||||
"semantic_string": "description: 'marisaundmarc\\'s story, 0 of 27, Unseen.', id context: 'avatar image view'",
|
||||
"y": 250, # Top story tray
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
intent = "profile picture avatar story ring"
|
||||
|
||||
# Should reject the user's own profile because clicking it means we edit/view our own story
|
||||
# instead of doing interactions with prospects.
|
||||
is_valid = engine._structural_sanity_check(own_story_node, intent, screen_height)
|
||||
|
||||
assert is_valid is False, "Structural Guard failed to reject the bot's OWN username story."
|
||||
|
||||
|
||||
def test_structural_reels_first_grid_item_y_coords():
|
||||
"""
|
||||
TDD Test: Reels viewer layout has grid items that are structurally valid.
|
||||
Ensures that relative Y coordinates (percentage of screen height) correctly
|
||||
allow valid grid items and block hallucinations.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Valid first grid item in a profile's reel tab, usually around y=700 to 1200
|
||||
valid_grid_node = {
|
||||
"semantic_string": "description: 'reel, 1 of 20', id context: 'image button'",
|
||||
"y": 800, # well within safe zone, ~33%
|
||||
"area": 40000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
# Hallucinated navigation tab node pretending to be "Home" around y=1200 (middle of screen)
|
||||
hallucinated_nav_node = {
|
||||
"semantic_string": "description: 'Home', id context: 'tab'",
|
||||
"y": 1200, # 50% height
|
||||
"area": 1000,
|
||||
"class_name": "android.view.View",
|
||||
}
|
||||
|
||||
intent_grid = "first grid item"
|
||||
intent_nav = "tap home tab"
|
||||
|
||||
is_valid_grid = engine._structural_sanity_check(valid_grid_node, intent_grid, screen_height)
|
||||
assert is_valid_grid is True, "Structural Guard rejected a valid reels grid item."
|
||||
|
||||
# The hallucinated nav node should be rejected because navigation tabs belong at the bottom!
|
||||
# Currently it might fail if we don't have relative coordinate checks!
|
||||
is_valid_nav = engine._structural_sanity_check(hallucinated_nav_node, intent_nav, screen_height)
|
||||
assert (
|
||||
is_valid_nav is False
|
||||
), "Structural Guard failed to reject a hallucinated navigation tab in the middle of the screen."
|
||||
|
||||
def test_structural_guard_rejects_search_keyword_for_media_content():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "text: 'i\\'m', id context: 'row search keyword title'",
|
||||
"class_name": "android.widget.TextView",
|
||||
"y": 500
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "post media content", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject 'row_search_keyword_title' for 'post media content'."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_search_user_for_post_username():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "desc: 'Followed by pratiek_the_entrepreneur + 19 more', id context: 'row search user container'",
|
||||
"class_name": "android.widget.LinearLayout",
|
||||
"y": 800
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "tap post username", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject 'row_search_user_container' for 'tap post username'."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_follow_button_for_author_username_header():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "text: 'Following', desc: 'Following Mariischen', id context: 'profile header follow button'",
|
||||
"class_name": "android.widget.Button",
|
||||
"y": 600
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "post author username header", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject follow button for 'post author username header'."
|
||||
@@ -14,7 +14,7 @@ class TestVerifySuccessGridReels:
|
||||
self.engine = TelepathicEngine()
|
||||
# Simulate a click context so verify_success has something to check against
|
||||
TelepathicEngine._last_click_context = {
|
||||
"intent": "first image in explore grid",
|
||||
"intent": "view a post",
|
||||
"semantic_string": "id context: 'image button'",
|
||||
"x": 178,
|
||||
"y": 558,
|
||||
@@ -32,7 +32,7 @@ class TestVerifySuccessGridReels:
|
||||
<node content-desc="Comment" resource-id="com.instagram.android:id/clips_comment_button" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", reel_xml)
|
||||
result = self.engine.verify_success("view a post", reel_xml)
|
||||
assert result is True, "verify_success rejected a valid Reel view opened from grid tap"
|
||||
|
||||
def test_normal_feed_post_still_accepted(self):
|
||||
@@ -44,7 +44,7 @@ class TestVerifySuccessGridReels:
|
||||
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="@testuser" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", feed_xml)
|
||||
result = self.engine.verify_success("view a post", feed_xml)
|
||||
assert result is True, "verify_success rejected a valid feed post opened from grid tap"
|
||||
|
||||
def test_explore_grid_still_visible_is_failure(self):
|
||||
@@ -57,17 +57,17 @@ class TestVerifySuccessGridReels:
|
||||
<node text="Search" resource-id="com.instagram.android:id/action_bar_search_edit_text" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", explore_xml)
|
||||
result = self.engine.verify_success("view a post", explore_xml)
|
||||
assert result is None, "verify_success should return None (inconclusive) when grid is still visible"
|
||||
|
||||
def test_profile_grid_reel_accepted(self):
|
||||
"""Profile grid → Reel must also be accepted."""
|
||||
TelepathicEngine._last_click_context["intent"] = "first image post in profile grid"
|
||||
TelepathicEngine._last_click_context["intent"] = "view a post"
|
||||
reel_xml = """
|
||||
<hierarchy>
|
||||
<node resource-id="com.instagram.android:id/clips_viewer_view_pager" />
|
||||
<node resource-id="com.instagram.android:id/reel_viewer_subtitle" text="Audio" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image post in profile grid", reel_xml)
|
||||
result = self.engine.verify_success("view a post", reel_xml)
|
||||
assert result is True, "verify_success rejected a Reel opened from profile grid"
|
||||
|
||||
Reference in New Issue
Block a user