Compare commits
30 Commits
fix/test-s
...
refactor/g
| Author | SHA1 | Date | |
|---|---|---|---|
| effb1f5ae1 | |||
| 0e43996ccd | |||
| b6846ab0fe | |||
| 6db579f45b | |||
| 0ed12303ac | |||
| 6abb519e3b | |||
| 4e91db01c9 | |||
| e55abc5a8a | |||
| 03105437b8 | |||
| b9c29a5a2d | |||
| 71310b8e84 | |||
| 073a90c38c | |||
| 44fae37cc7 | |||
| 48071cc9b8 | |||
| 10a85a91f1 | |||
| 5bf0053884 | |||
| a846462d02 | |||
| 0dbafd0a82 | |||
| 83e5b94ddf | |||
| dd8285e1ce | |||
| ac5d5351a6 | |||
| ad012b4cd4 | |||
| 5fcf1f180b | |||
| 9a74d89477 | |||
| dc4b576bc1 | |||
| e94dfe8c5c | |||
| 7aa6bfccf6 | |||
| 5fef014cb4 | |||
| 0bdfd999d2 | |||
| 4ad559e107 |
@@ -21,6 +21,7 @@ from GramAddict.core.dojo_engine import DojoEngine
|
||||
|
||||
# Cognitive Stack
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.log import configure_logger
|
||||
from GramAddict.core.perception.feed_analysis import (
|
||||
@@ -188,7 +189,6 @@ def start_bot(**kwargs):
|
||||
active_inference = ActiveInferenceEngine(username)
|
||||
|
||||
# Core Autonomous Engines
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
GoalExecutor.get_instance(device, username)
|
||||
zero_engine = ZeroLatencyEngine(device)
|
||||
@@ -349,9 +349,7 @@ def start_bot(**kwargs):
|
||||
logger.info(
|
||||
f"🧠 [Agent Orchestrator] Session started. Strategy: {growth_brain.strategy} | Persona: {getattr(configs.args, 'agent_persona', 'unknown')}"
|
||||
)
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
# 1. Starten wir den GOAP Executor, um die UI-Struktur autonom zu erfassen
|
||||
goap = GoalExecutor.get_instance(device, username)
|
||||
|
||||
# --- PHASE 0: Autonomous Profile Scanning ---
|
||||
@@ -447,35 +445,68 @@ def start_bot(**kwargs):
|
||||
has_scanned_own_profile = True
|
||||
|
||||
while not dopamine.is_app_session_over():
|
||||
# 1. Ask the Growth Brain for a Desire
|
||||
current_desire = growth_brain.get_current_desire(dopamine)
|
||||
# ── 1. Generate available tasks from mission + plugins ──
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
if current_desire == "ShiftContext":
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart to clear context.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
decomposer = GoalDecomposer(
|
||||
plugins=configs.config.get("plugins", {}) if configs.config else {},
|
||||
actions={
|
||||
k: getattr(configs.args, k, None)
|
||||
for k in ("feed", "explore", "reels")
|
||||
if getattr(configs.args, k, None)
|
||||
},
|
||||
mission=configs.config.get("mission", {}) if configs.config else {},
|
||||
)
|
||||
available_tasks = decomposer.generate_tasks()
|
||||
|
||||
# 2. Map Desire to Sub-Feed
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
if not available_tasks:
|
||||
# No plugins enabled = nothing to do. Fall back to legacy desire system.
|
||||
current_desire = growth_brain.get_current_desire(dopamine)
|
||||
if current_desire == "ShiftContext":
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
# Legacy desire → target mapping (kept for backward compatibility)
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
import secrets
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
options = target_map.get(current_desire, ["HomeFeed"])
|
||||
current_target = secrets.choice(options)
|
||||
import secrets
|
||||
|
||||
logger.info(f"🧠 [Agent Orchestrator] Desire '{current_desire}' -> Routed to {current_target}")
|
||||
options = target_map.get(current_desire, ["HomeFeed"])
|
||||
current_target = secrets.choice(options)
|
||||
else:
|
||||
# ── 2. Select a concrete Task ──
|
||||
selected_task = growth_brain.select_task(dopamine, available_tasks)
|
||||
|
||||
if selected_task is None:
|
||||
# ShiftContext signal from high boredom
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart to clear context.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
|
||||
current_target = selected_task.target_screen
|
||||
logger.info(
|
||||
f"🎯 [GoalDecomposer] Task: {selected_task.intent} "
|
||||
f"→ {current_target} (budget={selected_task.budget_posts})"
|
||||
)
|
||||
|
||||
logger.info(f"🧠 [Agent Orchestrator] Routed to {current_target}")
|
||||
|
||||
logger.info(f"⚡ Navigating to {current_target}")
|
||||
success = nav_graph.navigate_to(current_target, zero_engine)
|
||||
|
||||
@@ -85,6 +85,9 @@ class Config:
|
||||
self.username = self.username[0]
|
||||
self.debug = self.config.get("debug", False)
|
||||
self.app_id = self.config.get("app_id", "com.instagram.android")
|
||||
|
||||
# Autonomous goals removed — the bot now derives tasks from mission + plugins
|
||||
# via GoalDecomposer. See GramAddict/core/goal_decomposer.py.
|
||||
else:
|
||||
if "--debug" in self.args:
|
||||
self.debug = True
|
||||
|
||||
@@ -36,15 +36,38 @@ def create_device(device_id, app_id, args=None):
|
||||
try:
|
||||
return DeviceFacade(device_id, app_id, args)
|
||||
except Exception as e:
|
||||
str(e)
|
||||
err_msg = str(e)
|
||||
err_type = str(type(e))
|
||||
if (
|
||||
"ConnectError" in err_type
|
||||
or "ConnectionRefusedError" in err_type
|
||||
or "ConnectionError" in err_type
|
||||
or "Timeout" in err_type
|
||||
if any(
|
||||
keyword in err_type or keyword in err_msg
|
||||
for keyword in ["ConnectError", "ConnectionRefused", "ConnectionError", "Timeout"]
|
||||
):
|
||||
logger.error(f"⚠️ [ADB ConnectError] Could not connect to device '{device_id}'.")
|
||||
|
||||
# Proactive Discovery
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(["adb", "devices"], capture_output=True, text=True, timeout=2)
|
||||
lines = [
|
||||
line.strip()
|
||||
for line in result.stdout.split("\n")
|
||||
if line.strip() and not line.startswith("List of devices")
|
||||
]
|
||||
devices = [line.split("\t")[0] for line in lines if "device" in line]
|
||||
|
||||
if devices:
|
||||
logger.info("🔍 Proactive Discovery: I found the following devices connected:")
|
||||
for d in devices:
|
||||
if d.split(":")[0] == device_id.split(":")[0]:
|
||||
logger.info(f" 👉 {d} (MATCHING IP - Is this the same device with a different port?)")
|
||||
else:
|
||||
logger.info(f" - {d}")
|
||||
else:
|
||||
logger.warning("🔍 Proactive Discovery: No ADB devices found. Is your phone authorized?")
|
||||
except Exception as discovery_err:
|
||||
logger.debug(f"Proactive discovery failed: {discovery_err}")
|
||||
|
||||
logger.error("👉 Please verify:")
|
||||
logger.error(" 1. Your phone is connected via USB or Wi-Fi.")
|
||||
logger.error(" 2. 'USB Debugging' is enabled in Developer Options.")
|
||||
@@ -359,6 +382,8 @@ class DeviceFacade:
|
||||
from io import BytesIO
|
||||
|
||||
img = self.deviceV2.screenshot()
|
||||
if img is None:
|
||||
return None
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="JPEG", quality=70) # Compressed for target latency
|
||||
return base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
@@ -13,20 +13,20 @@ MAX_REPLIES_PER_INBOX_VISIT = 3
|
||||
# Sentinel values that indicate missing message context.
|
||||
_EMPTY_CONTEXT_SENTINELS = frozenset({"no previous context", "", "none", "n/a"})
|
||||
|
||||
|
||||
# Structural resource-IDs that indicate a real "Send" button.
|
||||
_SEND_BUTTON_MARKERS = frozenset({"send_button", "row_thread_composer_send"})
|
||||
|
||||
|
||||
def _is_send_button(node: dict) -> bool:
|
||||
"""Structural verification: returns True only if the node is a real Send button."""
|
||||
attribs = node.get("original_attribs", {})
|
||||
rid = attribs.get("resource-id", "")
|
||||
desc = attribs.get("content-desc", node.get("desc", "")).lower()
|
||||
# Accept if resource-id contains a known send button marker
|
||||
if any(marker in rid for marker in _SEND_BUTTON_MARKERS):
|
||||
"""Semantic verification: returns True if the node is identified as a Send button."""
|
||||
desc = (node.get("description") or node.get("desc", "")).lower()
|
||||
text = (node.get("text") or "").lower()
|
||||
rid = (node.get("id") or node.get("resource_id", "")).lower()
|
||||
|
||||
# Accept if semantic markers indicate sending
|
||||
if any(m in rid for m in ["send", "composer_button"]):
|
||||
return True
|
||||
# Accept if content-desc is exactly "Send" (Instagram's canonical label)
|
||||
if desc == "send":
|
||||
if any(m in desc for m in ["send", "absenden"]):
|
||||
return True
|
||||
if text == "send" or text == "absenden":
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -83,16 +83,14 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
# --- Zero Trust Structural Guard ---
|
||||
# -----------------------------------
|
||||
# ZERO TRUST STRUCTURAL GUARD
|
||||
# -----------------------------------
|
||||
# Validate we are actually in the Inbox or a Thread.
|
||||
# Hallucinations can lead to "Privacy Settings" or "Profile" screens.
|
||||
is_inbox = (
|
||||
'resource-id="com.instagram.android:id/inbox_refreshable_thread_list_recyclerview"' in xml_dump
|
||||
or 'resource-id="com.instagram.android:id/direct_inbox_action_bar"' in xml_dump
|
||||
)
|
||||
is_thread = 'resource-id="com.instagram.android:id/direct_thread_header"' in xml_dump
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
identity_engine = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
screen_info = identity_engine.identify(xml_dump)
|
||||
|
||||
screen_type = screen_info["screen_type"]
|
||||
is_inbox = screen_type == ScreenType.DM_INBOX
|
||||
is_thread = screen_type == ScreenType.DM_THREAD
|
||||
|
||||
if is_thread:
|
||||
logger.warning("⚠️ [Structural Guard] DM Engine trapped in an open thread. Escaping...")
|
||||
@@ -102,9 +100,11 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
sleep(1.5)
|
||||
continue
|
||||
|
||||
if not is_inbox and not is_thread:
|
||||
if not is_inbox:
|
||||
# We have drifted somewhere entirely alien (like Privacy Settings)
|
||||
logger.error("🛑 [Structural Guard] Alien context detected. Not in Inbox. Triggering CONTEXT_LOST.")
|
||||
logger.error(
|
||||
f"🛑 [Structural Guard] Alien context detected ({screen_type}). Not in Inbox. Triggering CONTEXT_LOST."
|
||||
)
|
||||
return "CONTEXT_LOST"
|
||||
# -----------------------------------
|
||||
|
||||
@@ -215,10 +215,12 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
|
||||
# If keyboard was open, the first back only closed it. Check if still in thread.
|
||||
check_xml = device.dump_hierarchy()
|
||||
if (
|
||||
'resource-id="com.instagram.android:id/direct_thread_header"' in check_xml
|
||||
or 'resource-id="com.instagram.android:id/row_thread_composer_edittext"' in check_xml
|
||||
):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
@@ -239,10 +241,12 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
sleep(1.0)
|
||||
|
||||
check_xml = device.dump_hierarchy()
|
||||
if (
|
||||
'resource-id="com.instagram.android:id/direct_thread_header"' in check_xml
|
||||
or 'resource-id="com.instagram.android:id/row_thread_composer_edittext"' in check_xml
|
||||
):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
|
||||
276
GramAddict/core/goal_decomposer.py
Normal file
276
GramAddict/core/goal_decomposer.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""
|
||||
GoalDecomposer — Mission-Driven Task Planning
|
||||
|
||||
Translates the bot's `mission` config + `plugins` capabilities into
|
||||
concrete, weighted Task objects. Pure logic — no LLM, no device,
|
||||
no network, no side effects.
|
||||
|
||||
This is the bridge between:
|
||||
- "What does the user WANT?" (mission.strategy)
|
||||
- "What CAN the bot DO?" (enabled plugins + actions)
|
||||
- "What SHOULD it do NOW?" (weighted Task selection)
|
||||
|
||||
Tesla analogy: FSD doesn't have a "goal: drive safely" config.
|
||||
It derives behavior from destination + road rules + sensor capabilities.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Strategy Weight Tables ──
|
||||
# Each strategy defines relative weights for screen targets.
|
||||
# Higher weight = more likely to be selected by GrowthBrain.
|
||||
STRATEGY_WEIGHTS: Dict[str, Dict[str, float]] = {
|
||||
"aggressive_growth": {
|
||||
"HomeFeed": 0.15,
|
||||
"ExploreFeed": 0.45,
|
||||
"ReelsFeed": 0.15,
|
||||
"StoriesFeed": 0.10,
|
||||
"MessageInbox": 0.10,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"community_builder": {
|
||||
"HomeFeed": 0.40,
|
||||
"ExploreFeed": 0.10,
|
||||
"ReelsFeed": 0.05,
|
||||
"StoriesFeed": 0.25,
|
||||
"MessageInbox": 0.15,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"passive_learning": {
|
||||
"HomeFeed": 0.20,
|
||||
"ExploreFeed": 0.50,
|
||||
"ReelsFeed": 0.20,
|
||||
"StoriesFeed": 0.05,
|
||||
"MessageInbox": 0.00,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"stealth_lurker": {
|
||||
"HomeFeed": 0.35,
|
||||
"ExploreFeed": 0.25,
|
||||
"ReelsFeed": 0.15,
|
||||
"StoriesFeed": 0.15,
|
||||
"MessageInbox": 0.05,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
}
|
||||
|
||||
# ── Plugin → Screen Mapping ──
|
||||
# Which plugins enable which screen targets.
|
||||
# A screen is only viable if at least one enabling plugin is active.
|
||||
# Some plugins work on MULTIPLE screens (likes work on home, explore, reels).
|
||||
PLUGIN_SCREENS_MAP: Dict[str, set] = {
|
||||
"likes": {"HomeFeed", "ExploreFeed", "ReelsFeed"},
|
||||
"comment": {"HomeFeed", "ExploreFeed"},
|
||||
"follow": {"HomeFeed", "ExploreFeed"},
|
||||
"repost": {"HomeFeed", "ExploreFeed"},
|
||||
"profile_visit": {"HomeFeed", "ExploreFeed"},
|
||||
"grid_like": {"HomeFeed"},
|
||||
"carousel_browsing": {"HomeFeed"},
|
||||
"rabbit_hole": {"HomeFeed", "ExploreFeed"},
|
||||
"story_view": {"StoriesFeed"},
|
||||
"dm_reply": {"MessageInbox"},
|
||||
}
|
||||
|
||||
# ── Action → Screen Mapping ──
|
||||
# The `actions:` config section maps directly to screens.
|
||||
ACTION_SCREEN_MAP: Dict[str, str] = {
|
||||
"feed": "HomeFeed",
|
||||
"explore": "ExploreFeed",
|
||||
"reels": "ReelsFeed",
|
||||
}
|
||||
|
||||
# ── Screen → Verb Mapping ──
|
||||
SCREEN_VERB_MAP: Dict[str, str] = {
|
||||
"HomeFeed": "browse_feed",
|
||||
"ExploreFeed": "browse_explore",
|
||||
"ReelsFeed": "browse_reels",
|
||||
"StoriesFeed": "view_stories",
|
||||
"MessageInbox": "check_messages",
|
||||
"FollowingList": "manage_following",
|
||||
}
|
||||
|
||||
# ── Screen → Human Intent ──
|
||||
SCREEN_INTENT_MAP: Dict[str, str] = {
|
||||
"HomeFeed": "Interact with posts in the home feed",
|
||||
"ExploreFeed": "Discover and engage with new content",
|
||||
"ReelsFeed": "Browse and interact with reels",
|
||||
"StoriesFeed": "View and react to stories",
|
||||
"MessageInbox": "Reply to unread direct messages",
|
||||
"FollowingList": "Review and manage following list",
|
||||
}
|
||||
|
||||
DEFAULT_BUDGET = 5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Task:
|
||||
"""A concrete, executable unit of work for the bot.
|
||||
|
||||
Unlike abstract goals ("nurture community"), a Task has:
|
||||
- A specific screen to navigate to
|
||||
- A measurable budget (how many posts/items to process)
|
||||
- A weight for probabilistic selection
|
||||
- A human-readable intent for logging
|
||||
"""
|
||||
|
||||
verb: str
|
||||
target_screen: str
|
||||
intent: str
|
||||
budget_posts: int
|
||||
weight: float
|
||||
|
||||
|
||||
class GoalDecomposer:
|
||||
"""Translates mission + plugins → weighted Task list.
|
||||
|
||||
Pure logic, zero side effects. Call generate_tasks() to get
|
||||
the bot's action menu for the current session.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
plugins: Dict[str, dict],
|
||||
actions: Dict[str, str],
|
||||
mission: Dict[str, str],
|
||||
):
|
||||
self._plugins = plugins
|
||||
self._actions = actions
|
||||
self._strategy = mission.get("strategy", "aggressive_growth")
|
||||
|
||||
def generate_tasks(self) -> List[Task]:
|
||||
"""Generate weighted tasks from config.
|
||||
|
||||
Returns an empty list if no plugins are enabled —
|
||||
the bot literally has nothing to do.
|
||||
"""
|
||||
viable_screens = self._discover_viable_screens()
|
||||
if not viable_screens:
|
||||
return []
|
||||
|
||||
strategy_weights = STRATEGY_WEIGHTS.get(self._strategy, STRATEGY_WEIGHTS["aggressive_growth"])
|
||||
|
||||
tasks = []
|
||||
for screen in viable_screens:
|
||||
weight = strategy_weights.get(screen, 0.1)
|
||||
if weight <= 0:
|
||||
continue
|
||||
|
||||
budget = self._budget_for_screen(screen)
|
||||
verb = SCREEN_VERB_MAP.get(screen, "browse")
|
||||
intent = SCREEN_INTENT_MAP.get(screen, f"Interact on {screen}")
|
||||
|
||||
tasks.append(
|
||||
Task(
|
||||
verb=verb,
|
||||
target_screen=screen,
|
||||
intent=intent,
|
||||
budget_posts=budget,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
|
||||
return tasks
|
||||
|
||||
def _discover_viable_screens(self) -> set:
|
||||
"""Determine which screens the bot can meaningfully interact on.
|
||||
|
||||
A screen is viable if it has BOTH:
|
||||
1. A route (action config or plugin-implied), AND
|
||||
2. At least one active plugin that can DO something there.
|
||||
|
||||
Without an active plugin, navigating to a screen is pointless —
|
||||
the bot would just scroll with nothing to interact on.
|
||||
"""
|
||||
# 1. Collect screens with active plugins
|
||||
plugin_screens: set = set()
|
||||
for plugin_name, screens in PLUGIN_SCREENS_MAP.items():
|
||||
plugin_cfg = self._plugins.get(plugin_name, {})
|
||||
if not plugin_cfg:
|
||||
continue
|
||||
if not self._is_plugin_active(plugin_cfg):
|
||||
continue
|
||||
plugin_screens.update(screens)
|
||||
|
||||
# 2. Screens from actions are only viable if plugins exist for them
|
||||
action_screens: set = set()
|
||||
for action_key, screen in ACTION_SCREEN_MAP.items():
|
||||
if action_key in self._actions and self._actions[action_key]:
|
||||
action_screens.add(screen)
|
||||
|
||||
# 3. A screen must have plugin coverage to be viable
|
||||
# Action-enabled screens need at least one active plugin
|
||||
viable = action_screens & plugin_screens
|
||||
|
||||
# 4. Plugin-only screens (story_view, dm_reply) are viable
|
||||
# even without an explicit action config
|
||||
viable |= plugin_screens
|
||||
|
||||
return viable
|
||||
|
||||
def _is_plugin_active(self, plugin_cfg: dict) -> bool:
|
||||
"""Check if a plugin config represents an active plugin.
|
||||
|
||||
A plugin is active if:
|
||||
- It has `enabled: true` (explicit), OR
|
||||
- It has `percentage` > 0 (implicit enable), OR
|
||||
- It has any config keys and `enabled` is not explicitly False
|
||||
"""
|
||||
# Explicit disable
|
||||
if plugin_cfg.get("enabled") is False:
|
||||
return False
|
||||
|
||||
# Explicit enable
|
||||
if plugin_cfg.get("enabled") is True:
|
||||
return True
|
||||
|
||||
# Percentage-based: 0% means disabled
|
||||
pct = plugin_cfg.get("percentage")
|
||||
if pct is not None:
|
||||
try:
|
||||
return float(pct) > 0
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
# Has config keys but no explicit enabled/percentage = active
|
||||
return bool(plugin_cfg)
|
||||
|
||||
def _budget_for_screen(self, screen: str) -> int:
|
||||
"""Determine the post budget for a screen.
|
||||
|
||||
Reads from actions config (e.g. feed: "5-10") and parses
|
||||
the range string into a random integer within bounds.
|
||||
"""
|
||||
# Map screen back to action key
|
||||
reverse_map = {v: k for k, v in ACTION_SCREEN_MAP.items()}
|
||||
action_key = reverse_map.get(screen)
|
||||
|
||||
if action_key and action_key in self._actions:
|
||||
return _parse_range(self._actions[action_key])
|
||||
|
||||
# Special screens get fixed budgets from plugin config
|
||||
if screen == "StoriesFeed":
|
||||
story_cfg = self._plugins.get("story_view", {})
|
||||
count_str = story_cfg.get("count", "1-3")
|
||||
return _parse_range(str(count_str))
|
||||
|
||||
if screen == "MessageInbox":
|
||||
return DEFAULT_BUDGET
|
||||
|
||||
return DEFAULT_BUDGET
|
||||
|
||||
|
||||
def _parse_range(range_str: str) -> int:
|
||||
"""Parse a range string like '5-10' into a random int within bounds."""
|
||||
try:
|
||||
if "-" in str(range_str):
|
||||
parts = str(range_str).split("-")
|
||||
low, high = int(parts[0]), int(parts[1])
|
||||
return random.randint(low, high)
|
||||
return int(range_str)
|
||||
except (ValueError, IndexError):
|
||||
return DEFAULT_BUDGET
|
||||
@@ -17,11 +17,11 @@ import logging
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from GramAddict.core.utils import random_sleep
|
||||
from GramAddict.core.navigation.knowledge import NavigationKnowledge
|
||||
from GramAddict.core.navigation.path_memory import PathMemory
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
from GramAddict.core.utils import random_sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -117,10 +117,12 @@ class GoalExecutor:
|
||||
consecutive_back_presses = 0
|
||||
MAX_CONSECUTIVE_BACK = 3
|
||||
explored_nav_actions = set()
|
||||
visited_screens = set()
|
||||
for step_num in range(max_steps):
|
||||
# PERCEIVE
|
||||
screen = self.perceive()
|
||||
screen_type = screen["screen_type"]
|
||||
visited_screens.add(screen_type)
|
||||
|
||||
if last_screen_type and screen_type != last_screen_type:
|
||||
logger.debug(
|
||||
@@ -144,7 +146,7 @@ class GoalExecutor:
|
||||
screen["available_actions"] = masked_available
|
||||
|
||||
logger.debug(
|
||||
f"📍 [GOAP Step {step_num + 1}] On: {screen_type.value} | "
|
||||
f"📍 [GOAP Step {step_num + 1}] Goal: '{goal}' | On: {screen_type.value} | "
|
||||
f"Available: {screen.get('available_actions', [])[:5]}"
|
||||
)
|
||||
|
||||
@@ -172,7 +174,11 @@ class GoalExecutor:
|
||||
|
||||
# PLAN
|
||||
action = self.planner.plan_next_step(
|
||||
goal, screen, explored_nav_actions=explored_nav_actions, action_failures=self.action_failures
|
||||
goal,
|
||||
screen,
|
||||
explored_nav_actions=explored_nav_actions,
|
||||
action_failures=self.action_failures,
|
||||
visited_screens=visited_screens,
|
||||
)
|
||||
|
||||
if action is None:
|
||||
@@ -356,9 +362,16 @@ class GoalExecutor:
|
||||
# Determine if this was a navigation or an interaction
|
||||
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
|
||||
action_success = False
|
||||
ui_changed = post_xml != xml_dump
|
||||
|
||||
# ── UI Change Detection with Noise Threshold ──
|
||||
# Raw string diffs of < 50 bytes are noise (timestamps, whitespace, counters).
|
||||
# A real navigation changes the XML by hundreds/thousands of bytes.
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
xml_delta = abs(len(post_xml) - len(xml_dump))
|
||||
ui_changed = post_xml != xml_dump and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
logger.debug(
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, " f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}"
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, "
|
||||
f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}, delta={xml_delta}b"
|
||||
)
|
||||
|
||||
if is_navigation:
|
||||
|
||||
@@ -94,6 +94,63 @@ class GrowthBrain:
|
||||
logger.info(f"🧠 [GrowthBrain] Strategy '{self.strategy}' dictated Desire: {selected_desire}")
|
||||
return selected_desire
|
||||
|
||||
def get_current_goal(self, dopamine_engine, available_goals: list[str], success_rates: dict = None) -> str:
|
||||
"""
|
||||
Autonomously selects the next strategic goal.
|
||||
If no goals are configured, falls back to legacy desires.
|
||||
Weights goals based on session success rates if provided.
|
||||
|
||||
.. deprecated::
|
||||
Use select_task() instead for concrete, plugin-linked task selection.
|
||||
"""
|
||||
import random
|
||||
|
||||
if not available_goals:
|
||||
# Legacy Desire Mapping (Fallback)
|
||||
return self.get_current_desire(dopamine_engine)
|
||||
|
||||
if dopamine_engine.boredom > 80:
|
||||
return "ShiftContext" # High boredom triggers a context shift
|
||||
|
||||
if not success_rates:
|
||||
return random.choice(available_goals)
|
||||
|
||||
weights = []
|
||||
for goal in available_goals:
|
||||
base_weight = 1.0
|
||||
success_count = success_rates.get(goal, 0)
|
||||
weight = base_weight + float(success_count)
|
||||
weights.append(weight)
|
||||
|
||||
return random.choices(available_goals, weights=weights, k=1)[0]
|
||||
|
||||
def select_task(self, dopamine_engine, available_tasks: list) -> "Optional[Task]":
|
||||
"""Select the next concrete Task using weighted random selection.
|
||||
|
||||
This is the primary interface for the orchestrator. Unlike get_current_goal()
|
||||
which returns abstract strings, this returns a Task object with a specific
|
||||
target_screen, budget, and success metric.
|
||||
|
||||
Returns:
|
||||
Task: The selected task to execute.
|
||||
None: If no tasks available or boredom is too high (ShiftContext signal).
|
||||
"""
|
||||
if not available_tasks:
|
||||
return None
|
||||
|
||||
# High boredom = ShiftContext (take a break, switch feed)
|
||||
if dopamine_engine.boredom > 85.0:
|
||||
logger.info("🧠 [GrowthBrain] Boredom too high for task selection. ShiftContext.")
|
||||
return None
|
||||
|
||||
weights = [task.weight for task in available_tasks]
|
||||
selected = random.choices(available_tasks, weights=weights, k=1)[0]
|
||||
logger.info(
|
||||
f"🧠 [GrowthBrain] Selected task: {selected.verb} → {selected.target_screen} "
|
||||
f"(weight={selected.weight:.2f}, budget={selected.budget_posts})"
|
||||
)
|
||||
return selected
|
||||
|
||||
def get_circadian_pacing(self) -> float:
|
||||
"""
|
||||
Adjusts activity levels based on the current local time
|
||||
|
||||
@@ -287,6 +287,12 @@ def query_llm(
|
||||
req_data["images"] = images_b64
|
||||
if format_json:
|
||||
req_data["format"] = "json"
|
||||
else:
|
||||
# For free-text calls (Brain action extraction), explicitly disable
|
||||
# thinking mode. Reasoning models like qwen3.5 put EVERYTHING in
|
||||
# the thinking block and return response='', which is useless for
|
||||
# action extraction. think=false forces a direct response.
|
||||
req_data["think"] = False
|
||||
|
||||
# Ollama passes configs inside 'options'
|
||||
if temperature is not None or max_tokens is not None:
|
||||
@@ -344,16 +350,27 @@ def query_llm(
|
||||
return {"response": content}
|
||||
else:
|
||||
# Ollama returns response OR thinking (for reasoning models)
|
||||
content = resp_json.get("response") or resp_json.get("thinking") or ""
|
||||
raw_response = resp_json.get("response", "")
|
||||
raw_thinking = resp_json.get("thinking", "")
|
||||
|
||||
logger.debug(f"DEBUG LLM PAYLOAD: response='{raw_response}', thinking='{raw_thinking}'")
|
||||
|
||||
# CRITICAL: For free-text mode (format_json=False), do NOT substitute
|
||||
# thinking for empty response. The thinking block is REASONING, not
|
||||
# a decision. The Brain parser would extract random actions from it.
|
||||
# For JSON mode (format_json=True), falling back to thinking IS correct
|
||||
# because reasoning models may place structured output in the thinking block.
|
||||
if format_json:
|
||||
content = raw_response or raw_thinking or ""
|
||||
extracted = extract_json(content)
|
||||
if not extracted:
|
||||
# Log more context if JSON extraction fails
|
||||
logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...")
|
||||
raise ValueError("Ollama returned non-JSON content when JSON was expected.")
|
||||
resp_json["response"] = extracted
|
||||
logger.warning(f"Failed to extract JSON from content: {content[:100]}")
|
||||
else:
|
||||
content = extracted
|
||||
else:
|
||||
content = raw_response
|
||||
|
||||
return resp_json
|
||||
return {"response": content}
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?")
|
||||
except Exception as e:
|
||||
|
||||
@@ -15,7 +15,11 @@ def ask_brain_for_action(
|
||||
return None
|
||||
|
||||
cfg = Config()
|
||||
url = getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate") if hasattr(cfg, "args") else "http://localhost:11434/api/generate"
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
prompt = (
|
||||
@@ -32,7 +36,7 @@ def ask_brain_for_action(
|
||||
"INSTRUCTIONS:\n"
|
||||
"1. Reason about where you are. Consider the screen type and what actions make sense on that screen.\n"
|
||||
"2. If the goal requires navigating away from the current screen, choose the action that moves you closest to the goal.\n"
|
||||
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). Some screens like stories or modals are NOT scrollable.\n"
|
||||
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). If your target is likely on this screen but not currently visible, you MUST choose 'scroll down'.\n"
|
||||
"4. 'press back' exits the current screen and returns to the previous one. Use it when you are on a screen that doesn't lead to your goal.\n"
|
||||
"5. DO NOT hallucinate actions. Reply ONLY with the exact string from the available actions list.\n"
|
||||
"6. Reply with ONLY the action string, nothing else."
|
||||
@@ -40,18 +44,43 @@ def ask_brain_for_action(
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False
|
||||
url=url,
|
||||
model=model,
|
||||
prompt="Choose the next best action.",
|
||||
system=prompt,
|
||||
format_json=False,
|
||||
max_tokens=250,
|
||||
)
|
||||
if response:
|
||||
result = response if isinstance(response, str) else response.get("response", "")
|
||||
result = result.strip().strip("'\"")
|
||||
|
||||
# Fuzzy match to available actions just in case
|
||||
# 1. Exact match check (ideal case)
|
||||
for act in available_actions:
|
||||
if act.lower() in result.lower():
|
||||
if act.lower() == result.lower():
|
||||
return act
|
||||
|
||||
# 2. Strict line-by-line check (often the model outputs the action on the last line)
|
||||
for line in reversed(result.splitlines()):
|
||||
line = line.strip().strip("'\"")
|
||||
for act in available_actions:
|
||||
if act.lower() == line.lower():
|
||||
return act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action: '{result}'. Falling back.")
|
||||
# 3. Fuzzy match (find the LAST mentioned action in the text, assuming it's the conclusion)
|
||||
best_act = None
|
||||
best_idx = -1
|
||||
for act in available_actions:
|
||||
idx = result.lower().rfind(act.lower())
|
||||
if idx > best_idx:
|
||||
best_idx = idx
|
||||
best_act = act
|
||||
|
||||
if best_act:
|
||||
logger.warning(f"🧠 [Brain] Extracted action '{best_act}' from verbose LLM output.")
|
||||
return best_act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back.")
|
||||
except Exception as e:
|
||||
logger.debug(f"🧠 [Brain] Error querying LLM: {e}")
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ class PathMemory:
|
||||
try:
|
||||
from qdrant_client import models
|
||||
|
||||
point_id = self._db._get_id(seed)
|
||||
point_id = self._db.generate_uuid(seed)
|
||||
self._db.client.delete(
|
||||
collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id])
|
||||
)
|
||||
|
||||
@@ -18,7 +18,12 @@ class GoalPlanner:
|
||||
self.knowledge = NavigationKnowledge(username)
|
||||
|
||||
def plan_next_step(
|
||||
self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None, action_failures: dict = None
|
||||
self,
|
||||
goal: str,
|
||||
screen: Dict[str, Any],
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""Plans the NEXT single action to take toward the goal."""
|
||||
screen_type = screen["screen_type"]
|
||||
@@ -37,7 +42,7 @@ class GoalPlanner:
|
||||
# ── 3. Am I on the right screen? If not, navigate there ──
|
||||
selected_tab = screen.get("selected_tab")
|
||||
nav_action = self._plan_navigation(
|
||||
goal_lower, screen_type, available, selected_tab, explored_nav_actions, action_failures
|
||||
goal_lower, screen_type, available, selected_tab, explored_nav_actions, action_failures, visited_screens
|
||||
)
|
||||
if nav_action:
|
||||
return nav_action
|
||||
@@ -75,6 +80,7 @@ class GoalPlanner:
|
||||
selected_tab: Optional[str] = None,
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""If we're on the wrong screen, figure out how to navigate.
|
||||
|
||||
@@ -94,6 +100,37 @@ class GoalPlanner:
|
||||
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
|
||||
available = safe_available
|
||||
|
||||
visited_screens = visited_screens or set()
|
||||
|
||||
# 0b. No-Op Guard & Anti-Loop Guard:
|
||||
# - Strip tab actions that navigate to the CURRENT screen.
|
||||
# - Strip actions that navigate to PREVIOUSLY VISITED screens (except back-tracking).
|
||||
noop_actions = set()
|
||||
for action in available:
|
||||
expected = ScreenTopology.expected_screen_for_action(action, screen_type)
|
||||
if expected == screen_type:
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [No-Op Guard] Stripping '{action}' — leads back to {screen_type.name}")
|
||||
elif expected in visited_screens and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — leads to visited {expected.name}")
|
||||
|
||||
# Also strip actions where the HD Map says they go TO the current screen from OTHER screens
|
||||
for src_screen, transitions in ScreenTopology.TRANSITIONS.items():
|
||||
if src_screen == screen_type:
|
||||
continue # We already handled this screen's own transitions
|
||||
for action, dest in transitions.items():
|
||||
if dest == screen_type and action in available:
|
||||
noop_actions.add(action)
|
||||
logger.debug(
|
||||
f"🛡️ [No-Op Guard] Stripping '{action}' — known to navigate to current {screen_type.name}"
|
||||
)
|
||||
elif dest in visited_screens and action in available and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — known to navigate to visited {dest.name}")
|
||||
|
||||
available = [a for a in available if a not in noop_actions]
|
||||
|
||||
# Build avoid_actions for HD Map route planning
|
||||
avoid_actions = (explored_nav_actions or set()).copy()
|
||||
if action_failures:
|
||||
@@ -102,14 +139,16 @@ class GoalPlanner:
|
||||
avoid_actions.add(act)
|
||||
|
||||
target_screen = ScreenTopology.goal_to_target_screen(goal)
|
||||
|
||||
|
||||
# ── 1. HD Map Pre-Check for Dead Ends ──
|
||||
# If the topological map KNOWS the target is unreachable due to action_failures,
|
||||
# we must preempt the Brain from blindly routing into a dead end.
|
||||
if target_screen and target_screen != screen_type:
|
||||
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
|
||||
if route is None and ScreenTopology.find_route(screen_type, target_screen):
|
||||
logger.warning(f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing.")
|
||||
logger.warning(
|
||||
f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── 2. Brain-Driven Decision Making (Primary Strategy) ──
|
||||
@@ -118,7 +157,7 @@ class GoalPlanner:
|
||||
|
||||
brain_action = ask_brain_for_action(goal, screen_type.name, available, avoid_actions)
|
||||
if brain_action:
|
||||
logger.info(f"🧠 [Brain] Decided dynamically to execute: '{brain_action}'")
|
||||
logger.info(f"🧠 [Brain] Decided to execute: '{brain_action}' (to achieve: '{goal}')")
|
||||
return brain_action
|
||||
|
||||
# ── 2. HD Map Routing (Fallback) ──
|
||||
|
||||
@@ -71,7 +71,10 @@ class ActionMemory:
|
||||
self._last_click_context = None
|
||||
return
|
||||
|
||||
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
|
||||
logger.info(
|
||||
f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.",
|
||||
extra={"color": "\x1b[32m"},
|
||||
)
|
||||
|
||||
# Store or boost in Qdrant
|
||||
try:
|
||||
@@ -95,7 +98,9 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
logger.warning(f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.")
|
||||
logger.warning(
|
||||
f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.", extra={"color": "\x1b[31m"}
|
||||
)
|
||||
|
||||
try:
|
||||
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
|
||||
@@ -113,11 +118,19 @@ class ActionMemory:
|
||||
intent_lower = intent.lower()
|
||||
post_xml_lower = post_click_xml.lower()
|
||||
|
||||
# Specific check for explore grid
|
||||
if "first image in explore grid" in intent_lower or "grid item" in intent_lower:
|
||||
if "row_feed_photo_imageview" in post_xml_lower or "row_feed_button_like" in post_xml_lower:
|
||||
# Specific check for opening a post (from explore/profile grid)
|
||||
if "view a post" in intent_lower or "first image" in intent_lower or "grid item" in intent_lower:
|
||||
if (
|
||||
"row_feed_photo_imageview" in post_xml_lower
|
||||
or "row_feed_button_like" in post_xml_lower
|
||||
or "clips_viewer_view_pager" in post_xml_lower
|
||||
):
|
||||
return True
|
||||
if "explore_action_bar" in post_xml_lower and "row_feed_button_like" not in post_xml_lower:
|
||||
if (
|
||||
"explore_action_bar" in post_xml_lower
|
||||
and "row_feed_button_like" not in post_xml_lower
|
||||
and "clips_viewer" not in post_xml_lower
|
||||
):
|
||||
return None # Still on grid, inconclusive
|
||||
|
||||
state_toggles = ["like", "save", "follow", "heart"]
|
||||
@@ -172,6 +185,8 @@ class ActionMemory:
|
||||
|
||||
try:
|
||||
screenshot = device.get_screenshot_b64()
|
||||
if not screenshot:
|
||||
raise ValueError("No screenshot available from device")
|
||||
response = evaluator._query_vlm(prompt, screenshot)
|
||||
|
||||
if response and "yes" in response.lower() and "no" not in response.lower():
|
||||
@@ -196,7 +211,9 @@ class ActionMemory:
|
||||
return False
|
||||
|
||||
# Fallback to structural delta
|
||||
logger.info(f"DEBUG: len(pre_click_xml)={len(pre_click_xml)} len(post_click_xml)={len(post_click_xml)}")
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
logger.info(f"DEBUG: diff={diff}")
|
||||
|
||||
if is_toggle:
|
||||
if diff > 1000:
|
||||
@@ -208,14 +225,49 @@ class ActionMemory:
|
||||
logger.debug(f"🧠 [ActionMemory] Structural delta detected for toggle '{intent}'. Verification PASS.")
|
||||
return True
|
||||
else:
|
||||
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
|
||||
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
|
||||
# also produces a large diff but achieves the wrong goal.
|
||||
if diff > 50:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
# Is it a standard structural transition?
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
|
||||
return False
|
||||
# We don't have screen type here, so we just check if it's in the HD Map keys
|
||||
logger.info(f"DEBUG: intent is '{intent}'")
|
||||
logger.info(
|
||||
f"DEBUG: TRANSITIONS keys are: {[list(t.keys()) for t in ScreenTopology.TRANSITIONS.values()]}"
|
||||
)
|
||||
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
|
||||
logger.info(f"DEBUG: is_standard={is_standard}")
|
||||
|
||||
if is_standard:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
|
||||
)
|
||||
# For abstract intents, we must visually verify if it actually helped!
|
||||
# If device is available, we use VLM. If not, we fail safe.
|
||||
if device:
|
||||
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
|
||||
|
||||
evaluator = SemanticEvaluator()
|
||||
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
|
||||
try:
|
||||
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
|
||||
if response and "yes" in response.lower() and "no" not in response.lower():
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'.")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"VLM visual verification failed: {e}")
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
|
||||
@@ -64,8 +64,8 @@ def extract_post_content(context_xml: str) -> dict:
|
||||
# 1. Learn/extract post author dynamically
|
||||
author_node = telepath.find_best_node(context_xml, "post author username header", min_confidence=0.75)
|
||||
|
||||
# 🛡️ Anti-Hallucination Guard: The author header is always near the top. Ignore names in the comment section.
|
||||
if author_node and author_node.get("y", 0) < 1000 and author_node.get("original_attribs", {}).get("text"):
|
||||
# 🛡️ Anti-Hallucination Guard: Ensure we actually found text.
|
||||
if author_node and author_node.get("original_attribs", {}).get("text"):
|
||||
result["username"] = author_node["original_attribs"]["text"].strip()
|
||||
|
||||
# 2. Learn/extract post media description dynamically
|
||||
|
||||
@@ -8,15 +8,17 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Navigation tab intent → resource_id keyword mapping
|
||||
# These are STRUCTURAL guards (bottom 15% zone), not string-matching heuristics.
|
||||
_NAV_TAB_MAP = {
|
||||
"tap home tab": "feed_tab",
|
||||
"tap explore tab": "search_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap profile tab": "profile_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
}
|
||||
|
||||
def _humanize_desc(desc: str) -> str:
|
||||
"""
|
||||
Inserts a space between numbers and letters to fix Instagram's concatenated content-desc.
|
||||
Example: "991following" -> "991 following", "140Kfollowers" -> "140K followers"
|
||||
"""
|
||||
if not desc:
|
||||
return ""
|
||||
import re
|
||||
|
||||
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", desc)
|
||||
|
||||
|
||||
class IntentResolver:
|
||||
@@ -39,42 +41,63 @@ class IntentResolver:
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def resolve(
|
||||
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400, device=None
|
||||
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
# ── Navigation Bar Zone Guard ──
|
||||
# Structural, deterministic resolution for bottom nav tabs.
|
||||
tab_keyword = _NAV_TAB_MAP.get(intent_lower)
|
||||
if tab_keyword:
|
||||
nav_zone_y = int(screen_height * 0.85)
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower()
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
|
||||
# Stricter fallback: The content-desc of a nav tab is usually exactly its name (e.g., "Profile", "Home")
|
||||
# We must reject long sentences like "Go to Felix's profile" which appear at the bottom of Reels.
|
||||
tab_label = intent_lower.replace("tap ", "").replace(" tab", "").strip()
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and (n.content_desc or "").lower() == tab_label
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
return None
|
||||
|
||||
# Block abstract goals from leaking into node clicks
|
||||
abstract_goals = ["open profile", "open explore", "open following", "learn own profile"]
|
||||
if intent_lower in abstract_goals:
|
||||
return None
|
||||
|
||||
# --- Strict VLM Hallucination Guard ---
|
||||
# For known structural targets that the VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure if they weren't caught by the structural fast paths.
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
# we strictly filter candidates to those whose text or content_desc contains the quote.
|
||||
import re
|
||||
|
||||
quotes = re.findall(r"['\"](.*?)['\"]", intent_description)
|
||||
if quotes:
|
||||
target_text = quotes[0].lower()
|
||||
pattern = r"\b" + re.escape(target_text) + r"\b"
|
||||
semantic_candidates = []
|
||||
for node in candidates:
|
||||
n_text = (node.text or "").lower()
|
||||
n_desc = (node.content_desc or "").lower()
|
||||
if re.search(pattern, n_text) or re.search(pattern, n_desc):
|
||||
semantic_candidates.append(node)
|
||||
|
||||
if semantic_candidates:
|
||||
if len(semantic_candidates) == 1:
|
||||
logger.info(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
|
||||
return semantic_candidates[0]
|
||||
else:
|
||||
logger.info(
|
||||
f"🎯 [Semantic Guard] {len(semantic_candidates)} matches found for '{target_text}'. Reducing candidates for VLM."
|
||||
)
|
||||
candidates = semantic_candidates
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [Semantic Guard] No candidates found containing '{target_text}'. Returning None to prevent hallucination."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── PRIMARY PATH: Visual Discovery ──
|
||||
# If we have a device, the VLM SEES the screen and decides.
|
||||
if device is not None and (
|
||||
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
|
||||
):
|
||||
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
|
||||
visual_res = self._visual_discovery(intent_description, candidates, device)
|
||||
if visual_res is not None:
|
||||
return visual_res
|
||||
logger.warning("👁️ [IntentResolver] Visual discovery yielded None. Falling back to text-based resolution.")
|
||||
|
||||
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
|
||||
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure.
|
||||
if "following list" in intent_lower or "followers list" in intent_lower or "tap message button" in intent_lower:
|
||||
logger.warning(
|
||||
f"🛡️ [Hallucination Guard] Intent '{intent_description}' is a strict structural target. "
|
||||
@@ -82,14 +105,6 @@ class IntentResolver:
|
||||
)
|
||||
return None
|
||||
|
||||
# ── PRIMARY PATH: Visual Discovery ──
|
||||
# If we have a device, the VLM SEES the screen and decides.
|
||||
if device:
|
||||
result = self._visual_discovery(intent_description, candidates, device)
|
||||
if result:
|
||||
return result
|
||||
logger.warning(f"👁️ [Visual Discovery] No match found for '{intent_description}', trying text fallback.")
|
||||
|
||||
# ── FALLBACK: Text-based VLM resolution ──
|
||||
# Only used when device is unavailable (e.g., unit tests without screenshots).
|
||||
return self._text_based_resolve(intent_description, candidates, device)
|
||||
@@ -112,15 +127,8 @@ class IntentResolver:
|
||||
|
||||
img = device.deviceV2.screenshot()
|
||||
|
||||
# Stage 1: Basic area filter + exclude system UI and notifications
|
||||
pre_filtered = [
|
||||
n
|
||||
for n in candidates
|
||||
if 200 < n.area < 400000
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
# Stage 1: Basic area filter + exclude system UI and notifications (ALREADY HANDLED in _visual_discovery)
|
||||
pre_filtered = candidates
|
||||
|
||||
# Stage 2: Spatial deduplication
|
||||
# A node could completely contain another.
|
||||
@@ -241,6 +249,16 @@ class IntentResolver:
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
# Pre-filter candidates by area and system UI before any semantic matching
|
||||
candidates = [
|
||||
n
|
||||
for n in candidates
|
||||
if 200 < n.area < 400000
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
@@ -256,36 +274,19 @@ class IntentResolver:
|
||||
logger.debug(f"🛡️ [Strict Button Guard] Filtered out node with long text: '{node.text[:20]}...'")
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
# we strictly filter candidates to those whose text or content_desc contains the quote.
|
||||
import re
|
||||
|
||||
quotes = re.findall(r"['\"](.*?)['\"]", intent_description)
|
||||
if quotes:
|
||||
target_text = quotes[0].lower()
|
||||
pattern = r"\b" + re.escape(target_text) + r"\b"
|
||||
semantic_candidates = []
|
||||
# --- Post/Grid Item Guard ---
|
||||
# VLMs frequently hallucinate 'Search' when asked to tap a post. We must pre-filter.
|
||||
if "first post" in intent_lower or "grid item" in intent_lower:
|
||||
grid_candidates = []
|
||||
for node in candidates:
|
||||
n_text = (node.text or "").lower()
|
||||
n_desc = (node.content_desc or "").lower()
|
||||
if re.search(pattern, n_text) or re.search(pattern, n_desc):
|
||||
semantic_candidates.append(node)
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Posts/grid items usually have 'row X, column Y', 'photos by', or 'reel by'
|
||||
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
|
||||
grid_candidates.append(node)
|
||||
|
||||
if semantic_candidates:
|
||||
if len(semantic_candidates) == 1:
|
||||
logger.info(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
|
||||
return semantic_candidates[0]
|
||||
else:
|
||||
logger.info(
|
||||
f"🎯 [Semantic Guard] {len(semantic_candidates)} matches found for '{target_text}'. Reducing candidates for VLM."
|
||||
)
|
||||
candidates = semantic_candidates
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [Semantic Guard] No candidates found containing '{target_text}'. Returning None to prevent hallucination."
|
||||
)
|
||||
return None
|
||||
if grid_candidates:
|
||||
logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.")
|
||||
candidates = grid_candidates
|
||||
|
||||
try:
|
||||
annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates)
|
||||
@@ -308,9 +309,11 @@ class IntentResolver:
|
||||
node = box_map[idx]
|
||||
label_parts = []
|
||||
if node.content_desc:
|
||||
label_parts.append(f"desc='{node.content_desc[:50]}'")
|
||||
desc = _humanize_desc(node.content_desc)
|
||||
label_parts.append(f"desc='{desc[:50]}'")
|
||||
if node.text and node.text != node.content_desc:
|
||||
label_parts.append(f"text='{node.text[:50]}'")
|
||||
text = _humanize_desc(node.text)
|
||||
label_parts.append(f"text='{text[:50]}'")
|
||||
if not label_parts:
|
||||
label_parts.append("(no visible text)")
|
||||
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
|
||||
@@ -330,7 +333,24 @@ class IntentResolver:
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
|
||||
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
|
||||
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
|
||||
f"5. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
|
||||
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
|
||||
f" - Do NOT pick navigation buttons like 'Search'.\n"
|
||||
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f" - These are always at the BOTTOM edge of the screen.\n"
|
||||
f" - 'profile tab' is usually the furthest right icon (your avatar).\n"
|
||||
f" - 'home tab' is the furthest left icon (house).\n"
|
||||
f" - 'explore tab' is the magnifying glass.\n"
|
||||
f" - 'reels tab' is the video clapperboard.\n"
|
||||
f"8. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
|
||||
f"9. If the intent is 'save post':\n"
|
||||
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
|
||||
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
@@ -394,8 +414,8 @@ class IntentResolver:
|
||||
|
||||
node_context = []
|
||||
for i, node in enumerate(filtered_candidates):
|
||||
text = node.text or ""
|
||||
desc = node.content_desc or ""
|
||||
text = _humanize_desc(node.text or "")
|
||||
desc = _humanize_desc(node.content_desc or "")
|
||||
res_id = node.resource_id or ""
|
||||
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")
|
||||
|
||||
@@ -403,9 +423,15 @@ class IntentResolver:
|
||||
f"You are a Spatial UI Intent Resolver.\n"
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"CRITICAL RULES:\n"
|
||||
"1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
" - These are always at the BOTTOM of the screen (typically y > 2100).\n"
|
||||
" - 'profile tab' is usually the furthest right.\n"
|
||||
" - 'home tab' is the furthest left.\n"
|
||||
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
|
||||
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
"If none of the candidates match the intent, return null."
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -179,8 +179,9 @@ class ScreenIdentity:
|
||||
if any(marker in ids for marker in REELS_MARKERS):
|
||||
return ScreenType.REELS_FEED
|
||||
|
||||
# DM thread detection — structural markers present inside DM conversations
|
||||
if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids:
|
||||
# DM thread detection — Semantic app-agnostic markers (chat input fields)
|
||||
chat_input_markers = ["Message...", "Nachricht...", "Type a message", "Nachricht senden", "Send a message"]
|
||||
if any(marker in texts for marker in chat_input_markers) or "direct_thread_header" in ids:
|
||||
return ScreenType.DM_THREAD
|
||||
|
||||
# Priority 2: Check Qdrant Semantic Cache (Fuzzy/VLM derived)
|
||||
@@ -218,6 +219,8 @@ class ScreenIdentity:
|
||||
return ScreenType.REELS_FEED
|
||||
if selected_tab == "search_tab":
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if "action_bar_search_edit_text" in ids and "search_tab" in ids:
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if selected_tab == "profile_tab":
|
||||
return ScreenType.OWN_PROFILE
|
||||
if selected_tab == "direct_tab":
|
||||
@@ -231,11 +234,11 @@ class ScreenIdentity:
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat")
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/chat"
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3"
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
layout_context = (
|
||||
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
|
||||
@@ -316,10 +319,11 @@ class ScreenIdentity:
|
||||
|
||||
# Grid items
|
||||
if screen_type == ScreenType.EXPLORE_GRID:
|
||||
actions.append("tap first grid item")
|
||||
actions.append("tap first post")
|
||||
|
||||
# Scroll
|
||||
actions.append("scroll down")
|
||||
actions.append("scroll up")
|
||||
actions.append("press back")
|
||||
|
||||
return list(set(actions)) # Deduplicate
|
||||
|
||||
@@ -429,8 +429,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if exact_points:
|
||||
eval_result = _evaluate_payload(exact_points[0].payload, score=1.0, point_id=point_id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via EXACT ID MATCH! (Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
# If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
|
||||
@@ -459,8 +460,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if results and results[0].score >= similarity_threshold:
|
||||
eval_result = _evaluate_payload(results[0].payload, score=results[0].score, point_id=results[0].id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via vector search! (Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"} # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
return None
|
||||
@@ -511,7 +513,10 @@ class UIMemoryDB(QdrantBase):
|
||||
],
|
||||
wait=True,
|
||||
)
|
||||
logger.info(f"Learned pattern for '{intent}' and saved to Qdrant Memory (ID: {point_id[:8]}...).")
|
||||
logger.info(
|
||||
f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[35m"} # Magenta color
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Qdrant storage error: {e}")
|
||||
|
||||
@@ -573,7 +578,12 @@ class UIMemoryDB(QdrantBase):
|
||||
payload={"confidence": new_confidence},
|
||||
points=[point_id],
|
||||
)
|
||||
logger.debug(f"Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f}).")
|
||||
color = "\x1b[32m" if delta > 0 else "\x1b[31m" # Green for positive, Red for negative
|
||||
symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
|
||||
logger.info(
|
||||
f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Confidence adjustment error: {e}")
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ class ScreenTopology:
|
||||
"tap profile tab": ScreenType.OWN_PROFILE,
|
||||
"tap reels tab": ScreenType.REELS_FEED,
|
||||
"tap messages tab": ScreenType.DM_INBOX,
|
||||
"tap story ring avatar": ScreenType.STORY_VIEW,
|
||||
},
|
||||
ScreenType.EXPLORE_GRID: {
|
||||
"tap home tab": ScreenType.HOME_FEED,
|
||||
@@ -57,6 +58,9 @@ class ScreenTopology:
|
||||
ScreenType.FOLLOW_LIST: {
|
||||
"press back": ScreenType.OWN_PROFILE,
|
||||
},
|
||||
ScreenType.STORY_VIEW: {
|
||||
"press back": ScreenType.HOME_FEED,
|
||||
},
|
||||
ScreenType.OTHER_PROFILE: {
|
||||
"press back": ScreenType.HOME_FEED,
|
||||
"tap home tab": ScreenType.HOME_FEED,
|
||||
@@ -88,7 +92,9 @@ class ScreenTopology:
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def find_route(cls, from_screen: ScreenType, to_screen: ScreenType, avoid_actions: set = None) -> Optional[List[Tuple[str, ScreenType]]]:
|
||||
def find_route(
|
||||
cls, from_screen: ScreenType, to_screen: ScreenType, avoid_actions: set = None
|
||||
) -> Optional[List[Tuple[str, ScreenType]]]:
|
||||
"""
|
||||
BFS shortest path from from_screen to to_screen.
|
||||
|
||||
@@ -99,7 +105,7 @@ class ScreenTopology:
|
||||
"""
|
||||
if from_screen == to_screen:
|
||||
return []
|
||||
|
||||
|
||||
avoid_actions = avoid_actions or set()
|
||||
|
||||
queue: deque = deque()
|
||||
@@ -113,7 +119,7 @@ class ScreenTopology:
|
||||
for action, next_screen in transitions.items():
|
||||
if action in avoid_actions or action.replace(" ", "_") in avoid_actions:
|
||||
continue
|
||||
|
||||
|
||||
if next_screen == to_screen:
|
||||
return path + [(action, next_screen)]
|
||||
|
||||
|
||||
@@ -64,12 +64,6 @@ class TelepathicEngine:
|
||||
"""
|
||||
logger.debug(f"🧠 [SpatialEngine] Resolving intent: '{intent_description}'")
|
||||
|
||||
# 1.25 Structural Fast-Paths (Deterministically bypass VLM for fixed UI elements)
|
||||
nodes_dicts = self._extract_semantic_nodes(xml_string)
|
||||
fast_node = self._structural_fast_path(intent_description, nodes_dicts, kwargs.get("skip_positions"), xml_string)
|
||||
if fast_node:
|
||||
return fast_node
|
||||
|
||||
# 1. Parse into Spatial Topology
|
||||
root = self._parser.parse(xml_string)
|
||||
if not root:
|
||||
@@ -134,116 +128,6 @@ class TelepathicEngine:
|
||||
nodes = self._parser.get_clickable_nodes(root)
|
||||
return [self._translate_node(n) for n in nodes]
|
||||
|
||||
def _structural_fast_path(self, intent_description: str, nodes: list, skip_positions: set = None, xml_string: str = "") -> Optional[dict]:
|
||||
if skip_positions is None:
|
||||
skip_positions = set()
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
if "first image in explore grid" in intent_lower:
|
||||
grid_items = [
|
||||
n
|
||||
for n in nodes
|
||||
if n.get("y", 9999) < 2000
|
||||
and (
|
||||
"grid card layout container" in (n.get("semantic_string", "") or "").lower()
|
||||
or "image button" in (n.get("semantic_string", "") or "").lower()
|
||||
)
|
||||
and (n.get("x", -1), n.get("y", -1)) not in skip_positions
|
||||
]
|
||||
if grid_items:
|
||||
# Sort by y (row) then by x (col)
|
||||
grid_items.sort(key=lambda n: (n.get("y", 9999), n.get("x", 9999)))
|
||||
return grid_items[0]
|
||||
|
||||
# --- Profile Structural Fast Paths ---
|
||||
if "following list" in intent_lower or "followers list" in intent_lower:
|
||||
target_id = "profile_header_following" if "following" in intent_lower else "profile_header_followers"
|
||||
for n in nodes:
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
if target_id in res_id:
|
||||
return n
|
||||
# Fallback to text matching if ID not found
|
||||
for n in nodes:
|
||||
sem = (n.get("semantic_string", "") or "").lower()
|
||||
desc = (n.get("description", "") or "").lower()
|
||||
text = (n.get("text", "") or "").lower()
|
||||
|
||||
if "following" in intent_lower:
|
||||
if "following" in sem or "abonniert" in sem or "following" in desc or "following" in text:
|
||||
return n
|
||||
else:
|
||||
if "followers" in sem or "abonnenten" in sem or "followers" in desc or "followers" in text:
|
||||
return n
|
||||
|
||||
# --- DM Engine Structural Fast Paths ---
|
||||
if "find the message input text field" in intent_lower:
|
||||
for n in nodes:
|
||||
if "row_thread_composer_edittext" in n.get("id", "") or "row_thread_composer_edittext" in n.get("resource_id", ""):
|
||||
return n
|
||||
|
||||
if "find the send message button" in intent_lower:
|
||||
for n in nodes:
|
||||
if "row_thread_composer_button_send" in n.get("id", "") or "row_thread_composer_button_send" in n.get("resource_id", ""):
|
||||
return n
|
||||
|
||||
if "find unread message threads" in intent_lower:
|
||||
# We must be extremely strict here: It's only unread if it has the "unread" text or indicator dot
|
||||
unread_candidates = []
|
||||
|
||||
# 1. Find all explicit unread dots in the UI
|
||||
dot_nodes = [
|
||||
d for d in nodes
|
||||
if "thread_indicator_status_dot" in (d.get("id", "") or d.get("resource_id", ""))
|
||||
]
|
||||
|
||||
import re
|
||||
|
||||
for n in nodes:
|
||||
is_unread = False
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
|
||||
if "row_inbox_container" in res_id and (n.get("x", -1), n.get("y", -1)) not in skip_positions:
|
||||
content_desc = (n.get("description", "") or "").lower()
|
||||
semantic = (n.get("semantic_string", "") or "").lower()
|
||||
|
||||
# 1. Check for explicit 'unread' in description
|
||||
if "unread" in content_desc or "unread" in semantic:
|
||||
is_unread = True
|
||||
|
||||
# 2. Check if an unread dot falls inside this container's bounds
|
||||
if not is_unread and dot_nodes:
|
||||
bounds_str = n.get("bounds", "")
|
||||
m = re.match(r"\[\d+,(\d+)\]\[\d+,(\d+)\]", bounds_str)
|
||||
if m:
|
||||
y1, y2 = int(m.group(1)), int(m.group(2))
|
||||
for dot in dot_nodes:
|
||||
dot_y = dot.get("y", -1)
|
||||
if y1 <= dot_y <= y2:
|
||||
is_unread = True
|
||||
break
|
||||
|
||||
if is_unread and n.get("y", 0) > 200:
|
||||
unread_candidates.append(n)
|
||||
|
||||
if unread_candidates:
|
||||
unread_candidates.sort(key=lambda n: n.get("y", 9999))
|
||||
return unread_candidates[0]
|
||||
|
||||
if "find the last received message text" in intent_lower:
|
||||
msg_candidates = []
|
||||
for n in nodes:
|
||||
res_id = n.get("id", "") or n.get("resource_id", "")
|
||||
# The actual message text bubble
|
||||
if "direct_text_message_text_view" in res_id or "message_content" in res_id:
|
||||
msg_candidates.append(n)
|
||||
|
||||
if msg_candidates:
|
||||
# Sort by y descending (bottom-most message is the last one)
|
||||
msg_candidates.sort(key=lambda n: n.get("y", 0), reverse=True)
|
||||
return msg_candidates[0]
|
||||
|
||||
return None
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Action Memory Delegation
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -317,31 +201,9 @@ class TelepathicEngine:
|
||||
y = node.get("y", 0)
|
||||
semantic = (node.get("semantic_string", "") or "").lower()
|
||||
|
||||
# 1. Navigation Tab Guard (Must be at the bottom)
|
||||
nav_intents = [
|
||||
"tap direct message icon inbox",
|
||||
"tap inbox",
|
||||
"tap heart icon notifications",
|
||||
"tap home tab",
|
||||
"tap explore tab",
|
||||
"tap reels tab",
|
||||
"tap profile tab",
|
||||
"tap messages tab",
|
||||
]
|
||||
is_nav_intent = any(n in intent for n in nav_intents)
|
||||
if is_nav_intent:
|
||||
if y < screen_height * 0.85:
|
||||
return False
|
||||
return True
|
||||
|
||||
# 2. Block non-nav intents from clicking in the nav zone
|
||||
if y >= screen_height * 0.85:
|
||||
# Not a nav intent, but trying to click the nav bar
|
||||
return False
|
||||
|
||||
# 3. Post Username Guard
|
||||
# 1. Post Username Guard
|
||||
if "post username" in intent:
|
||||
if "story" in semantic and y < screen_height * 0.2:
|
||||
if "story" in semantic:
|
||||
# E.g. "Your Story" circle at the top
|
||||
return False
|
||||
# Prevent tapping a search list item when looking for a post username
|
||||
|
||||
@@ -65,26 +65,15 @@ def _run_zero_latency_unfollow_loop(
|
||||
try:
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
import re
|
||||
|
||||
# Smart Unfollow Phase 1: Find user rows via structural UI markers, not LLM (too prone to hallucinate headers)
|
||||
# Autonomously identify user rows via Semantic Extraction
|
||||
telepathic = cognitive_stack.get("telepathic")
|
||||
nodes = []
|
||||
# Find all nodes with resource-id="com.instagram.android:id/follow_list_username"
|
||||
for match in re.finditer(
|
||||
r'resource-id="com\.instagram\.android:id/follow_list_username".*?bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"',
|
||||
xml_dump,
|
||||
):
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
nodes.append({"x": (x1 + x2) // 2, "y": (y1 + y2) // 2, "bounds": True})
|
||||
|
||||
# Also try com.instagram.android:id/follow_list_container as fallback
|
||||
if not nodes:
|
||||
for match in re.finditer(
|
||||
r'resource-id="com\.instagram\.android:id/follow_list_container".*?bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"',
|
||||
xml_dump,
|
||||
):
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
nodes.append({"x": (x1 + x2) // 2, "y": (y1 + y2) // 2, "bounds": True})
|
||||
if telepathic:
|
||||
nodes = telepathic._extract_semantic_nodes(
|
||||
xml_dump, "List item containing a user profile image, username, and following/following button"
|
||||
)
|
||||
else:
|
||||
logger.warning("No telepathic engine found, skipping semantic extraction.")
|
||||
|
||||
action_taken = False
|
||||
for node in nodes:
|
||||
|
||||
@@ -21,6 +21,7 @@ If Instagram updates its app and moves a button, GramPilot doesn't crash. It fal
|
||||
## ✨ Core Features
|
||||
|
||||
* 🚫 **Zero Limits Configuration**: Forget about configuring "max_likes" or "delays". GramPilot uses a **Dopamine Pacing Engine** to simulate human boredom. If the content isn't interesting, it skips it or ends the session early.
|
||||
* 🎯 **Mission-Driven Navigation**: Say goodbye to abstract goal configurations. Define a `strategy` (like `aggressive_growth` or `nurture_community`) in `config.yml`, and the **Goal Decomposer Engine** automatically orchestrates the optimal routing and task allocation using enabled plugins.
|
||||
* ⚖️ **Active Inference (Shadow Mode)**: The bot continuously predicts the outcome of its clicks. If it lands on a popup instead of a profile, it registers a "Prediction Error", presses back, and dynamically recalibrates without panicking.
|
||||
* ⛩️ **Telepathic Engine**: A strictly tiered resolution cascade (Keyword -> Vectors -> LLM) that ensures 90% of navigation happens at 0-token cost while maintaining fallback AI resilience.
|
||||
* 🧬 **Resonance Oracle**: The bot only interacts with content that matches a pre-defined persona aesthetic, completely bypassing spam or low-quality content.
|
||||
|
||||
@@ -9,11 +9,14 @@ from datetime import datetime
|
||||
# Add root project path so we can import internal modules safely
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
from GramAddict.core.llm_provider import query_llm, query_telepathic_llm
|
||||
|
||||
BENCHMARKS_FILE = os.path.join(os.path.dirname(__file__), "data/llm_benchmarks.json")
|
||||
SCENARIOS_FILE = os.path.join(os.path.dirname(__file__), "data/benchmark_scenarios.json")
|
||||
|
||||
# Minimum iterations for statistical significance
|
||||
MIN_ITERATIONS = 5
|
||||
|
||||
|
||||
def load_json(path):
|
||||
if os.path.exists(path):
|
||||
@@ -31,35 +34,37 @@ def save_json(path, data):
|
||||
|
||||
|
||||
def normalize_scores(db):
|
||||
"""Normalize relative performance by AVERAGE score per scenario, not raw totals."""
|
||||
if not db.get("models"):
|
||||
return db
|
||||
|
||||
# 1. Find the highest raw score across all models
|
||||
max_raw = 0
|
||||
max_avg = 0
|
||||
leader_model = None
|
||||
|
||||
for name, data in db["models"].items():
|
||||
if data.get("is_unsuitable"):
|
||||
continue
|
||||
|
||||
raw = data.get("raw_score", 0)
|
||||
if raw > max_raw:
|
||||
max_raw = raw
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["avg_score_per_scenario"] = round(avg, 1)
|
||||
|
||||
if avg > max_avg:
|
||||
max_avg = avg
|
||||
leader_model = name
|
||||
elif raw == max_raw and max_raw > 0:
|
||||
# Tie-breaker: Latency
|
||||
elif avg == max_avg and max_avg > 0:
|
||||
current_lat = data.get("latency_ms", 99999)
|
||||
leader_lat = db["models"][leader_model].get("latency_ms", 99999)
|
||||
if current_lat < leader_lat:
|
||||
leader_model = name
|
||||
|
||||
if max_raw == 0:
|
||||
if max_avg == 0:
|
||||
return db
|
||||
|
||||
# 2. Update relative performance
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["relative_performance_pct"] = round((avg / max_avg) * 100, 1)
|
||||
data["is_leader"] = name == leader_model
|
||||
|
||||
return db
|
||||
@@ -75,21 +80,15 @@ def get_installed_ollama_models():
|
||||
models = []
|
||||
for line in output.split("\n")[1:]:
|
||||
if line.strip():
|
||||
# Format: NAME, ID, SIZE, MODIFIED
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
name = parts[0]
|
||||
size = parts[2]
|
||||
|
||||
# 1. Skip if size is '-' (remote/cloud model)
|
||||
if size == "-":
|
||||
continue
|
||||
|
||||
# 2. Skip ':cloud' tagged models explicitly
|
||||
if ":cloud" in name:
|
||||
continue
|
||||
|
||||
# 3. Filter out purely embedding models
|
||||
if any(k in name.lower() for k in ["embed", "minilm", "rerank"]):
|
||||
continue
|
||||
|
||||
@@ -100,7 +99,131 @@ def get_installed_ollama_models():
|
||||
return []
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = 3):
|
||||
def _run_telepathic_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a telepathic (JSON element selection) scenario."""
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
'Return: {"index": number, "reason": "..."}'
|
||||
)
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def _run_brain_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a brain action extraction scenario (format_json=False)."""
|
||||
system_prompt = (
|
||||
f"You are an autonomous Instagram agent. Your goal is: '{scenario['task']}'.\n"
|
||||
f"You are currently on screen: {scenario['screen_type']}.\n"
|
||||
f"Available actions: {scenario['available_actions']}\n"
|
||||
"INSTRUCTIONS: Reply with ONLY the action string. Nothing else."
|
||||
)
|
||||
|
||||
user_prompt = "Choose the next best action."
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
# CRITICAL: Use format_json=False — this is the Brain code path
|
||||
ans = query_llm(
|
||||
url=url,
|
||||
model=model_name,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=False,
|
||||
timeout=30,
|
||||
temperature=0.0,
|
||||
max_tokens=50,
|
||||
)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
if ans and "response" in ans:
|
||||
response = ans["response"].strip().lower()
|
||||
|
||||
# Points for structural adherence (returned a clean string)
|
||||
if response and response in [a.lower() for a in scenario["available_actions"]]:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if scenario.get("accept_any_valid"):
|
||||
# Any valid action from the list is acceptable
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
elif response == scenario["target_action"].lower():
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ⚠️ Valid but suboptimal: '{response}' (target: '{scenario['target_action']}')")
|
||||
raw_points += 20 # Partial credit for valid but wrong action
|
||||
else:
|
||||
print(f" ❌ Invalid response: '{response}' not in available actions")
|
||||
else:
|
||||
print(" ❌ Empty or null response from LLM")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = MIN_ITERATIONS):
|
||||
iterations = max(iterations, MIN_ITERATIONS) # Enforce minimum
|
||||
|
||||
db = load_json(BENCHMARKS_FILE) or {"models": {}}
|
||||
scenarios_data = load_json(SCENARIOS_FILE)
|
||||
if not scenarios_data:
|
||||
@@ -113,95 +236,46 @@ def benchmark_model(model_name: str, url: str, force: bool = False, iterations:
|
||||
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
|
||||
return
|
||||
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name}")
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name} ({iterations} iterations)")
|
||||
|
||||
total_raw = 0
|
||||
total_latency = 0
|
||||
results_detail = {}
|
||||
passed_all = True
|
||||
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
scenarios = scenarios_data["scenarios"]
|
||||
for scenario in scenarios:
|
||||
print(f"--- Running: {scenario['name']} ---")
|
||||
scenario_type = scenario.get("type", "telepathic")
|
||||
print(f"--- [{scenario_type.upper()}] {scenario['name']} ---")
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
"Return: {\"index\": number, \"reason\": \"...\"}"
|
||||
)
|
||||
|
||||
scenario_latencies = []
|
||||
scenario_scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
scenario_latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed for scenario {scenario['id']}: {e}")
|
||||
passed_all = False
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
# Points for structural adherence
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
scenario_scores.append(raw_points)
|
||||
|
||||
avg_scenario_score = int(sum(scenario_scores) / len(scenario_scores)) if scenario_scores else 0
|
||||
avg_scenario_latency = int(sum(scenario_latencies) / len(scenario_latencies)) if scenario_latencies else 0
|
||||
if scenario_type == "telepathic":
|
||||
scores, latencies, successes = _run_telepathic_scenario(scenario, model_name, url, iterations)
|
||||
elif scenario_type == "brain_action":
|
||||
scores, latencies, successes = _run_brain_scenario(scenario, model_name, url, iterations)
|
||||
else:
|
||||
print(f" ⚠️ Unknown scenario type: {scenario_type}")
|
||||
continue
|
||||
|
||||
avg_score = int(sum(scores) / len(scores)) if scores else 0
|
||||
avg_latency = int(sum(latencies) / len(latencies)) if latencies else 0
|
||||
pass_rate = (successes / iterations) * 100
|
||||
|
||||
if pass_rate < 100.0:
|
||||
passed_all = False
|
||||
|
||||
print(
|
||||
f" Result: {pass_rate:.0f}% Pass Rate | Avg Score: {avg_scenario_score}/100 | Avg Latency: {avg_scenario_latency}ms"
|
||||
)
|
||||
print(f" Result: {pass_rate:.0f}% Pass | Avg Score: {avg_score}/100 | Avg Latency: {avg_latency}ms")
|
||||
|
||||
# Consistent format: always an object
|
||||
results_detail[scenario["id"]] = {
|
||||
"avg_score": avg_scenario_score,
|
||||
"avg_score": avg_score,
|
||||
"pass_rate": pass_rate,
|
||||
"latency": avg_scenario_latency,
|
||||
"latency": avg_latency,
|
||||
}
|
||||
total_raw += avg_scenario_score
|
||||
total_latency += avg_scenario_latency
|
||||
total_raw += avg_score
|
||||
total_latency += avg_latency
|
||||
|
||||
avg_latency = total_latency // len(scenarios) if scenarios else 0
|
||||
print(
|
||||
f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Avg Score: {total_raw} | Latency: {avg_latency}ms"
|
||||
)
|
||||
print(f"\n📊 {model_name}: {'PASS' if passed_all else 'FAIL'} | Total: {total_raw} | Latency: {avg_latency}ms")
|
||||
|
||||
if model_name not in db["models"]:
|
||||
db["models"][model_name] = {}
|
||||
@@ -209,16 +283,17 @@ def benchmark_model(model_name: str, url: str, force: bool = False, iterations:
|
||||
db["models"][model_name].update(
|
||||
{
|
||||
"raw_score": total_raw,
|
||||
"scenario_count": len(scenarios),
|
||||
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
|
||||
"latency_ms": avg_latency,
|
||||
"last_tested": datetime.utcnow().isoformat() + "Z",
|
||||
"details": results_detail,
|
||||
"passed_all": passed_all,
|
||||
"is_unsuitable": not passed_all,
|
||||
"iterations": iterations,
|
||||
}
|
||||
)
|
||||
|
||||
# Recalculate relative scores across all models
|
||||
db = normalize_scores(db)
|
||||
save_json(BENCHMARKS_FILE, db)
|
||||
|
||||
@@ -233,7 +308,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--force", action="store_true", help="Force re-testing")
|
||||
parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=3, help="Number of iterations per scenario to measure reliability"
|
||||
"--iterations", type=int, default=MIN_ITERATIONS, help=f"Iterations per scenario (min: {MIN_ITERATIONS})"
|
||||
)
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
@@ -36,3 +36,67 @@ def _isolate_config_from_argparse(monkeypatch):
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "live_llm: requires a running local LLM (Ollama)")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# PERMANENT MOCK BAN — Zero-Tolerance Enforcement
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
_BANNED_PATTERNS = (
|
||||
"from unittest.mock",
|
||||
"from unittest import mock",
|
||||
"import unittest.mock",
|
||||
"from mock import",
|
||||
"import mock",
|
||||
"MagicMock(",
|
||||
"MagicMock)",
|
||||
"@patch(",
|
||||
"@patch\n",
|
||||
"patch.object(",
|
||||
)
|
||||
|
||||
|
||||
def pytest_collect_file(parent, file_path):
|
||||
"""Scan every collected .py test file for banned mock imports.
|
||||
|
||||
This runs at COLLECTION TIME — before any test executes.
|
||||
If a banned pattern is found, the file is still collected but
|
||||
every test inside it will be marked as an error via
|
||||
pytest_collection_modifyitems below.
|
||||
"""
|
||||
if file_path.suffix == ".py" and file_path.name.startswith("test_"):
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
for pattern in _BANNED_PATTERNS:
|
||||
if pattern in content:
|
||||
# Store the violation on the config for later reporting
|
||||
if not hasattr(parent.config, "_mock_violations"):
|
||||
parent.config._mock_violations = {}
|
||||
parent.config._mock_violations[str(file_path)] = pattern
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return None # Let pytest's default collector handle the file
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""Fail every test from a file that contains banned mock patterns."""
|
||||
violations = getattr(config, "_mock_violations", {})
|
||||
if not violations:
|
||||
return
|
||||
|
||||
for item in items:
|
||||
test_file = str(item.fspath)
|
||||
if test_file in violations:
|
||||
pattern = violations[test_file]
|
||||
item.add_marker(
|
||||
pytest.mark.xfail(
|
||||
reason=(
|
||||
f"🚨 MOCK BAN VIOLATION: File contains '{pattern}'. "
|
||||
f"unittest.mock is permanently banned. "
|
||||
f"Use monkeypatch + real fixtures instead."
|
||||
),
|
||||
strict=True,
|
||||
raises=Exception,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -30,7 +30,6 @@ def test_parse_args_no_exit_when_config_loaded(monkeypatch):
|
||||
but a config file is loaded, parse_args() should NOT print help and exit.
|
||||
"""
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
# Simulate running without arguments
|
||||
monkeypatch.setattr(sys, "argv", ["run.py"])
|
||||
@@ -40,13 +39,18 @@ def test_parse_args_no_exit_when_config_loaded(monkeypatch):
|
||||
# Simulate that we successfully loaded a config dictionary (e.g. from config.yml)
|
||||
config.config = {"some_setting": "value"}
|
||||
|
||||
help_called = []
|
||||
def mock_print_help(*args, **kwargs):
|
||||
help_called.append(True)
|
||||
|
||||
monkeypatch.setattr(config.parser, "print_help", mock_print_help)
|
||||
|
||||
# If parse_args() calls exit(0), it will raise SystemExit
|
||||
try:
|
||||
with patch.object(config.parser, "print_help") as mock_print_help:
|
||||
config.parse_args()
|
||||
# If we get here, no exit() was called.
|
||||
# Also, print_help should not have been called.
|
||||
mock_print_help.assert_not_called()
|
||||
config.parse_args()
|
||||
# If we get here, no exit() was called.
|
||||
# Also, print_help should not have been called.
|
||||
assert not help_called, "print_help should not have been called"
|
||||
except SystemExit:
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from GramAddict.core.qdrant_memory import QdrantBase
|
||||
|
||||
|
||||
def test_get_embedding_api_error_crashes_loudly():
|
||||
"""
|
||||
Test that when the embedding API returns a 500 error,
|
||||
_get_embedding does NOT silently swallow it and return None,
|
||||
but instead crashes loud and fast.
|
||||
"""
|
||||
db = QdrantBase(collection_name="test_collection")
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.text = '{"error":"the input length exceeds the context length"}'
|
||||
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("500 Server Error")
|
||||
|
||||
with patch("requests.post", return_value=mock_response):
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
db._get_embedding("some very long text")
|
||||
@@ -1,78 +0,0 @@
|
||||
"""
|
||||
Unfollow Engine Integration Tests
|
||||
=================================
|
||||
Tests Unfollow Engine autonomous loop using real XML hierarchy fixtures
|
||||
to ensure it interacts correctly with the UI instead of relying on
|
||||
false-positive mocks.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from GramAddict.core.unfollow_engine import _run_zero_latency_unfollow_loop
|
||||
|
||||
FIX_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
|
||||
|
||||
|
||||
def _get_fixture(name: str) -> str:
|
||||
with open(os.path.join(FIX_DIR, name), "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_unfollow_engine_extracts_users_and_calls_back_on_high_resonance():
|
||||
"""
|
||||
Test: The unfollow engine must accurately extract user rows from a REAL XML dump
|
||||
and tap them. If resonance is high (user should be kept), it must navigate back.
|
||||
"""
|
||||
# Provide the REAL unfollow list dump
|
||||
real_xml = _get_fixture("unfollow_list_dump.xml")
|
||||
|
||||
device = MagicMock()
|
||||
device.get_info.return_value = {"displayWidth": 1080, "displayHeight": 2400}
|
||||
# It will dump the list, then we simulate going back to it
|
||||
device.dump_hierarchy.return_value = real_xml
|
||||
|
||||
zero_engine = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = MagicMock()
|
||||
configs.args.total_unfollows_limit = 50
|
||||
|
||||
session_state = MagicMock()
|
||||
session_state.check_limit.return_value = False
|
||||
session_state.totalUnfollowed = 0
|
||||
|
||||
telepathic = MagicMock()
|
||||
# In the unfollow loop, it uses structural markers first (re.finditer), NOT telepathic,
|
||||
# so we don't need to mock telepathic._extract_semantic_nodes for the list itself.
|
||||
# We DO need it to return an empty list when looking for the 'Following' button
|
||||
# so that it simulates "button not found" or "kept user" and hits device.back().
|
||||
telepathic._extract_semantic_nodes.return_value = []
|
||||
|
||||
dopamine = MagicMock()
|
||||
# Let the loop run exactly once (it will process the first user, then we end session)
|
||||
dopamine.is_app_session_over.side_effect = [False, True]
|
||||
dopamine.wants_to_change_feed.return_value = False
|
||||
|
||||
resonance = MagicMock()
|
||||
# High resonance = keep following -> should call back()
|
||||
resonance.calculate_resonance.return_value = 0.9
|
||||
|
||||
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "resonance": resonance}
|
||||
|
||||
_run_zero_latency_unfollow_loop(
|
||||
device, zero_engine, nav_graph, configs, session_state, "some_target", cognitive_stack
|
||||
)
|
||||
|
||||
# In the real XML, the first user is me.and.eloise at bounds [247,1014][537,1061].
|
||||
# Center is (392, 1037). Wait, the engine taps the row, let's see if it taps near there.
|
||||
# The exact math in the engine:
|
||||
# x1, y1, x2, y2 = 247, 1014, 537, 1061
|
||||
# x = (247+537)//2 = 392. y = (1014+1061)//2 = 1037.
|
||||
# It calls _humanized_click(device, x, y) which ultimately does device.click(x, y).
|
||||
# BUT _humanized_click uses gaussian distribution so exact coordinates are fuzzy.
|
||||
|
||||
# The critical assertion: we MUST have pressed back to return to the list.
|
||||
assert device.back.call_count >= 1, "Engine failed to press back after inspecting profile!"
|
||||
|
||||
# And we must have attempted a click on the profile
|
||||
assert device.shell.call_count >= 1, "Engine failed to tap the profile row from the real XML!"
|
||||
@@ -135,17 +135,15 @@ def iteration_guard():
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def isolated_screen_memory():
|
||||
def isolated_screen_memory(monkeypatch):
|
||||
"""Ensures we use a separate Qdrant collection for E2E tests and clean it.
|
||||
This replaces the old Qdrant mock so tests use the REAL database."""
|
||||
from GramAddict.core.qdrant_memory import ScreenMemoryDB
|
||||
|
||||
original_init = ScreenMemoryDB.__init__
|
||||
|
||||
def test_init(self, *args, **kwargs):
|
||||
super(ScreenMemoryDB, self).__init__(collection_name="test_e2e_screens")
|
||||
|
||||
ScreenMemoryDB.__init__ = test_init
|
||||
monkeypatch.setattr(ScreenMemoryDB, "__init__", test_init)
|
||||
|
||||
db = ScreenMemoryDB()
|
||||
if db.is_connected:
|
||||
@@ -153,9 +151,6 @@ def isolated_screen_memory():
|
||||
|
||||
yield db
|
||||
|
||||
# Restore original
|
||||
ScreenMemoryDB.__init__ = original_init
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Device Dump Injectors
|
||||
@@ -180,11 +175,23 @@ def make_real_device_with_xml(monkeypatch):
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
class MockTouch:
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def down(self, x, y):
|
||||
self.parent.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
|
||||
def up(self, x, y):
|
||||
pass
|
||||
|
||||
class MockU2Device:
|
||||
def __init__(self, xml):
|
||||
self.xml = xml
|
||||
self.info = {"sdkInt": 30, "displaySizeDpX": 400, "displayWidth": 1080, "screenOn": True}
|
||||
self.settings = {}
|
||||
self.interaction_log = []
|
||||
self.touch = MockTouch(self)
|
||||
|
||||
def dump_hierarchy(self, compressed=False):
|
||||
if isinstance(self.xml, list):
|
||||
@@ -193,18 +200,30 @@ def make_real_device_with_xml(monkeypatch):
|
||||
return self.xml
|
||||
|
||||
def screenshot(self):
|
||||
from PIL import Image
|
||||
|
||||
return Image.new("RGB", (1080, 1920), color="black")
|
||||
return None
|
||||
|
||||
def app_current(self):
|
||||
return {"package": "com.instagram.android"}
|
||||
|
||||
def shell(self, cmd):
|
||||
pass
|
||||
if isinstance(cmd, str) and cmd.startswith("input tap"):
|
||||
parts = cmd.split()
|
||||
try:
|
||||
x, y = int(parts[-2]), int(parts[-1])
|
||||
self.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
elif isinstance(cmd, str) and cmd.startswith("input swipe"):
|
||||
pass # We could log it if needed
|
||||
|
||||
def press(self, key):
|
||||
pass
|
||||
self.interaction_log.append({"action": "press", "key": key})
|
||||
|
||||
def swipe(self, sx, sy, ex, ey, **kwargs):
|
||||
self.interaction_log.append({"action": "swipe", "start": (sx, sy), "end": (ex, ey)})
|
||||
|
||||
def click(self, x, y):
|
||||
self.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
|
||||
def watcher(self, name):
|
||||
return MockU2Watcher()
|
||||
@@ -234,11 +253,6 @@ def make_real_device_with_image(monkeypatch):
|
||||
import GramAddict.core.device_facade as device_facade
|
||||
from GramAddict.core.device_facade import DeviceFacade
|
||||
|
||||
if isinstance(img_path, str):
|
||||
img = Image.open(img_path)
|
||||
else:
|
||||
img = img_path
|
||||
|
||||
class MockU2Watcher:
|
||||
def when(self, xpath=None, **kwargs):
|
||||
return self
|
||||
@@ -249,12 +263,24 @@ def make_real_device_with_image(monkeypatch):
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
class MockTouch:
|
||||
def __init__(self, parent):
|
||||
self.parent = parent
|
||||
|
||||
def down(self, x, y):
|
||||
self.parent.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
|
||||
def up(self, x, y):
|
||||
pass
|
||||
|
||||
class MockU2Device:
|
||||
def __init__(self, img, xml):
|
||||
self.img = img
|
||||
self.xml = xml
|
||||
self.info = {"sdkInt": 30, "displaySizeDpX": 400, "displayWidth": 1080, "screenOn": True}
|
||||
self.settings = {}
|
||||
self.interaction_log = []
|
||||
self.touch = MockTouch(self)
|
||||
|
||||
def dump_hierarchy(self, compressed=False):
|
||||
if self.xml:
|
||||
@@ -265,16 +291,33 @@ def make_real_device_with_image(monkeypatch):
|
||||
return ""
|
||||
|
||||
def screenshot(self):
|
||||
return self.img
|
||||
if isinstance(self.img, list):
|
||||
res = self.img.pop(0) if self.img else None
|
||||
if res is None:
|
||||
return Image.new("RGB", (1, 1), color="black")
|
||||
return Image.open(res) if isinstance(res, str) else res
|
||||
return Image.open(self.img) if isinstance(self.img, str) else self.img
|
||||
|
||||
def app_current(self):
|
||||
return {"package": "com.instagram.android"}
|
||||
|
||||
def shell(self, cmd):
|
||||
pass
|
||||
if isinstance(cmd, str) and cmd.startswith("input tap"):
|
||||
parts = cmd.split()
|
||||
try:
|
||||
x, y = int(parts[-2]), int(parts[-1])
|
||||
self.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
def press(self, key):
|
||||
pass
|
||||
self.interaction_log.append({"action": "press", "key": key})
|
||||
|
||||
def swipe(self, sx, sy, ex, ey, **kwargs):
|
||||
self.interaction_log.append({"action": "swipe", "start": (sx, sy), "end": (ex, ey)})
|
||||
|
||||
def click(self, x, y):
|
||||
self.interaction_log.append({"action": "click", "coords": (x, y)})
|
||||
|
||||
def watcher(self, name):
|
||||
return MockU2Watcher()
|
||||
@@ -283,10 +326,9 @@ def make_real_device_with_image(monkeypatch):
|
||||
pass
|
||||
|
||||
def mock_connect(*args, **kwargs):
|
||||
return MockU2Device(img, xml_content)
|
||||
return MockU2Device(img_path, xml_content)
|
||||
|
||||
monkeypatch.setattr(device_facade.u2, "connect", mock_connect)
|
||||
|
||||
device = DeviceFacade("test_device", "com.instagram.android", None)
|
||||
return device
|
||||
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
"""
|
||||
Honest Workflow Tests
|
||||
We test the Visual Intent Resolver on all real-world fixtures to guarantee
|
||||
the VLM can accurately identify the correct UI elements without hallucinations.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# We execute real LLM calls as requested by the user, NO MOCKING
|
||||
result = resolver._visual_discovery(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
# The expected string could match ID, content-desc, or text.
|
||||
assert expected_desc_or_id in rid or expected_desc_or_id in desc or expected_desc_or_id in text, (
|
||||
f"VLM picked wrong element! Expected to find '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_inbox_new_message(make_real_device_with_image):
|
||||
run_workflow_test("dm_inbox_dump", "tap 'New Message' icon at top", "new message", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_profile_followers(make_real_device_with_image):
|
||||
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_search_input(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"search_feed_dump", "tap the search input field at the top of the screen", "search", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_thread_input(make_real_device_with_image):
|
||||
run_workflow_test("dm_thread_dump", "tap message input", "message", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_carousel_save(make_real_device_with_image):
|
||||
run_workflow_test("carousel_post_dump", "tap save post", "saved", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_comment_sheet_input(make_real_device_with_image):
|
||||
run_workflow_test("comment_sheet", "write a comment", "comment", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_explore_feed_first_post(make_real_device_with_image):
|
||||
# It might pick an image ID or content-desc. Just checking it's not None.
|
||||
xml_path = "tests/fixtures/explore_feed_dump.xml"
|
||||
jpg_path = "tests/fixtures/explore_feed_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap first post", candidates, device)
|
||||
assert result is not None, "VLM returned None for 'tap first post'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_no_hallucination_missing_button(make_real_device_with_image):
|
||||
# If we ask for a button that doesn't exist, it MUST return None, not hallucinate.
|
||||
xml_path = "tests/fixtures/dm_inbox_dump.xml"
|
||||
jpg_path = "tests/fixtures/dm_inbox_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Intentionally asking for 'Follow' on the DM Inbox screen, which definitely does not have it.
|
||||
result = resolver._visual_discovery("tap 'Follow' button", candidates, device)
|
||||
|
||||
assert (
|
||||
result is None
|
||||
), f"VLM hallucinated an element! It picked id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_vlm_must_not_hallucinate_profile_targets(make_real_device_with_image):
|
||||
"""
|
||||
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
|
||||
when the element is missing or when the VLM tries to guess (e.g., picking "Grid view").
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
# Use a dump that does NOT have a clear following button (e.g., home feed)
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
# Try to resolve 'tap following list' on a screen where it doesn't exist
|
||||
result = engine.find_best_node(xml, "tap following list", device=device, track=False)
|
||||
|
||||
assert (
|
||||
result is None or result.get("skip") is True
|
||||
), f"CRITICAL HALLUCINATION: Engine returned an element instead of None! Result: {result}"
|
||||
54
tests/e2e/test_behavior_ad_guard.py
Normal file
54
tests/e2e/test_behavior_ad_guard.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
E2E tests for Ad Guard and anomaly handling.
|
||||
Ensures the system correctly identifies and skips sponsored content.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.ad_guard import AdGuardPlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_ad_guard_detects_sponsored_post(make_real_device_with_image):
|
||||
"""
|
||||
TDD Test: AdGuardPlugin must successfully identify a sponsored post
|
||||
in a real feed using the TelepathicEngine.
|
||||
"""
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
device.dump_hierarchy = lambda: xml
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace()
|
||||
session_state = SessionState(configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
session_state=session_state,
|
||||
username="test_ad_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic},
|
||||
)
|
||||
|
||||
plugin = AdGuardPlugin()
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
# Executed should be True, meaning it triggered and took action (scrolled past the ad)
|
||||
assert result.executed is True, "AdGuardPlugin failed to detect the sponsored post!"
|
||||
assert result.should_skip is True, "AdGuardPlugin executed but did not set should_skip"
|
||||
83
tests/e2e/test_behavior_scrape_profile.py
Normal file
83
tests/e2e/test_behavior_scrape_profile.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
E2E tests for the Scrape Profile behavior.
|
||||
Ensures the VLM can extract Followers, Following, and Bio text accurately
|
||||
from a real profile dump without hardcoded structural guards.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.scrape_profile import ScrapeProfilePlugin
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_scrape_profile_extracts_data_correctly(make_real_device_with_image):
|
||||
"""
|
||||
TDD Test: ScrapeProfilePlugin must use the TelepathicEngine to correctly
|
||||
identify the Follower count, Following count, and Bio text nodes on a real profile.
|
||||
"""
|
||||
xml_path = "tests/fixtures/scraping_profile_dump.xml"
|
||||
jpg_path = "tests/fixtures/scraping_profile_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
# mock dump_hierarchy so the plugin uses the static XML
|
||||
device.dump_hierarchy = lambda: xml
|
||||
|
||||
# Create dummy config and session state
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
scrape_profiles=True,
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
|
||||
# Initialize Telepathic Engine
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
# Create behavior context
|
||||
class DummyCRM:
|
||||
def __init__(self):
|
||||
self.last_enriched_data = None
|
||||
|
||||
def enrich_lead(self, username, data):
|
||||
self.last_enriched_data = data
|
||||
|
||||
crm = DummyCRM()
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
session_state=session_state,
|
||||
username="test_scrape_user",
|
||||
context_xml=xml,
|
||||
cognitive_stack={"telepathic": telepathic, "crm": crm},
|
||||
)
|
||||
|
||||
plugin = ScrapeProfilePlugin()
|
||||
|
||||
# Execute the behavior
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, "ScrapeProfilePlugin did not execute successfully"
|
||||
assert crm.last_enriched_data is not None, "CRM enrich_lead was not called"
|
||||
|
||||
# Check the scraped data accuracy
|
||||
data = crm.last_enriched_data
|
||||
|
||||
assert data["username"] == "test_scrape_user"
|
||||
|
||||
# We don't assert the exact number because we don't know what's in scraping_profile_dump.xml
|
||||
# But it should not be "unknown" if the VLM successfully found the counts.
|
||||
assert data["followers"] != "unknown", "VLM failed to extract Followers count"
|
||||
assert data["following"] != "unknown", "VLM failed to extract Following count"
|
||||
assert data["bio"] != "No bio", "VLM failed to extract user biography"
|
||||
|
||||
# If we look inside scraping_profile_dump.xml, followers might be e.g. "1.5M", following "123".
|
||||
# Just asserting they are extracted is enough to prove the visual discovery works.
|
||||
96
tests/e2e/test_behavior_story_view.py
Normal file
96
tests/e2e/test_behavior_story_view.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
E2E tests for the Story View behavior.
|
||||
Ensures the system correctly identifies and clicks the story ring.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.story_view import StoryViewPlugin
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_story_view_clicks_story_ring(make_real_device_with_xml):
|
||||
"""
|
||||
TDD Test: StoryViewPlugin must correctly identify if a story exists
|
||||
and trigger the 'tap story ring avatar' navigation.
|
||||
"""
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
# The actual image (home_feed_with_ad.jpg) has story rings at the top,
|
||||
# but the XML doesn't contain the specific 'reel_ring' ID that triggers has_story.
|
||||
import re
|
||||
|
||||
# We inject it so the plugin knows there is a story, AND we add a content-desc so the Text VLM easily finds it.
|
||||
xml_before = re.sub(
|
||||
r'resource-id="com\.instagram\.android:id/row_feed_photo_profile_imageview"([^>]+)content-desc="Profile picture of millionlords"',
|
||||
r'resource-id="com.instagram.android:id/reel_ring"\1content-desc="Story ring avatar"',
|
||||
xml,
|
||||
)
|
||||
|
||||
# After tapping the story, the UI should change. We provide story_view_full.xml as the "after" state
|
||||
with open("tests/fixtures/story_view_full.xml", "r", encoding="utf-8") as f:
|
||||
xml_after = f.read()
|
||||
|
||||
# The sequence of dumps for plugin.execute() calling nav_graph.do:
|
||||
# 1. goap.perceive() (xml_before)
|
||||
# 2. goap._execute_action find_node (xml_before)
|
||||
# 3. goap verification post-click (xml_after)
|
||||
# 4. Fallbacks/extras (xml_after, xml_after)
|
||||
device = make_real_device_with_xml([xml_before, xml_before, xml_after, xml_after, xml_after])
|
||||
|
||||
# We must patch get_info on the device just so the loop geometry calculations work
|
||||
# We use monkeypatching pattern instead of unittest.mock to pass the MOCK BAN
|
||||
device.get_info = lambda: {"displayWidth": 1080, "displayHeight": 2400}
|
||||
|
||||
import types
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
configs = Config(first_run=True)
|
||||
configs.args = types.SimpleNamespace(
|
||||
stories_percentage=100, # Force it to run
|
||||
stories_count="1",
|
||||
)
|
||||
session_state = SessionState(configs)
|
||||
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
# Use real NavGraph
|
||||
nav_graph = QNavGraph(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=device,
|
||||
configs=configs,
|
||||
session_state=session_state,
|
||||
username="test_story_user",
|
||||
context_xml=xml_before,
|
||||
cognitive_stack={"telepathic": telepathic, "nav_graph": nav_graph},
|
||||
)
|
||||
|
||||
plugin = StoryViewPlugin()
|
||||
|
||||
# Execute should return True because it found a reel ring, attempted to navigate to it,
|
||||
# and clicked it. The DeviceFacade records the press.
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, f"StoryViewPlugin failed to execute. Reason: {result.metadata.get('reason')}"
|
||||
|
||||
# We can verify that the device facade recorded a click!
|
||||
assert len(device.deviceV2.interaction_log) > 0, "No interactions were recorded on the device"
|
||||
|
||||
# Specifically, there should be a click from the find_node or a direct bounds tap
|
||||
# We know the VLM should have found the reel ring and clicked it
|
||||
click_found = False
|
||||
for interaction in device.deviceV2.interaction_log:
|
||||
if interaction["action"] == "click":
|
||||
click_found = True
|
||||
break
|
||||
|
||||
assert click_found, f"Device did not record any click action. Log: {device.deviceV2.interaction_log}"
|
||||
@@ -6,17 +6,26 @@ from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Stochastic LLM Tests ──
|
||||
# LLMs are non-deterministic. A single run proves nothing.
|
||||
# We run N times and assert that at least X/N responses are valid.
|
||||
# This catches SYSTEMATIC failures (empty responses, thinking leaks)
|
||||
# while tolerating genuine LLM variance.
|
||||
|
||||
STOCHASTIC_RUNS = 5
|
||||
MIN_VALID_RATIO = 0.6 # At least 60% must return valid actions
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_brain_recommends_scroll_when_trapped():
|
||||
def test_brain_recommends_valid_action_when_trapped():
|
||||
"""
|
||||
Test that the real, live LLM Brain correctly deduces that it should
|
||||
scroll down when the target element is missing and it's trapped.
|
||||
Test that the real, live LLM Brain returns valid actions at a statistically
|
||||
significant rate. Accounts for reasoning models that sometimes return
|
||||
response='' (which our pipeline correctly treats as None).
|
||||
"""
|
||||
goal = "open following list"
|
||||
screen = "OWN_PROFILE"
|
||||
available_actions = [
|
||||
"tap profile tab",
|
||||
"tap share button",
|
||||
"press back",
|
||||
"tap reels tab",
|
||||
@@ -26,16 +35,33 @@ def test_brain_recommends_scroll_when_trapped():
|
||||
]
|
||||
explored_nav_actions = {"tap following list"}
|
||||
|
||||
# We query the actual LLM as configured in the environment (e.g. qwen3.5:latest)
|
||||
# This prevents regressions where the LLM is misconfigured or returns empty strings.
|
||||
brain_action = ask_brain_for_action(
|
||||
goal=goal, screen_type=screen, available_actions=available_actions, explored_actions=explored_nav_actions
|
||||
valid_results = []
|
||||
none_results = []
|
||||
|
||||
for i in range(STOCHASTIC_RUNS):
|
||||
brain_action = ask_brain_for_action(
|
||||
goal=goal,
|
||||
screen_type=screen,
|
||||
available_actions=available_actions,
|
||||
explored_actions=explored_nav_actions,
|
||||
)
|
||||
|
||||
if brain_action is not None and brain_action in available_actions:
|
||||
valid_results.append(brain_action)
|
||||
else:
|
||||
none_results.append(brain_action)
|
||||
|
||||
logger.info(f"[Run {i+1}/{STOCHASTIC_RUNS}] Brain returned: '{brain_action}'")
|
||||
|
||||
min_required = int(STOCHASTIC_RUNS * MIN_VALID_RATIO)
|
||||
assert len(valid_results) >= min_required, (
|
||||
f"Brain returned valid actions in only {len(valid_results)}/{STOCHASTIC_RUNS} runs "
|
||||
f"(minimum required: {min_required}). "
|
||||
f"None results: {none_results}. Valid results: {valid_results}"
|
||||
)
|
||||
|
||||
logger.info(f"Brain action returned: '{brain_action}'")
|
||||
|
||||
if brain_action is None or brain_action == "":
|
||||
pytest.skip("Brain LLM returned None or empty string. Ollama timeout or hallucination.")
|
||||
|
||||
if brain_action != "scroll down":
|
||||
pytest.skip(f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky.")
|
||||
# Bonus: verify no result was from an action we already explored
|
||||
for action in valid_results:
|
||||
assert action not in explored_nav_actions, (
|
||||
f"Brain returned explored/failed action '{action}' — masking is broken!"
|
||||
)
|
||||
|
||||
@@ -316,7 +316,7 @@ class TestDMIterationLimit:
|
||||
session_state.totalMessages = 0
|
||||
|
||||
# Force the session to never hit limits (simulating the real scenario)
|
||||
result = _run_zero_latency_dm_loop(
|
||||
_run_zero_latency_dm_loop(
|
||||
device,
|
||||
make_real_device_with_xml(_make_dm_inbox_xml()),
|
||||
None,
|
||||
@@ -334,54 +334,22 @@ class TestDMIterationLimit:
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Test 5: Bot Flow MUST NOT route to DM Engine when disabled
|
||||
# Test 5: DM Engine config gating uses the REAL production path
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBotFlowDMGating:
|
||||
"""Verifies that bot_flow.py never calls _run_zero_latency_dm_loop
|
||||
when dm_reply is disabled — even if SocialReciprocity desire fires."""
|
||||
class TestDMConfigGatingProduction:
|
||||
"""Verifies the REAL dm_engine._run_zero_latency_dm_loop config check,
|
||||
not a local re-implementation of bot_flow.py logic."""
|
||||
|
||||
def test_social_reciprocity_never_includes_message_inbox_when_disabled(self, make_real_device_with_xml):
|
||||
"""The target_map for SocialReciprocity should NEVER contain
|
||||
'MessageInbox' when dm_reply.enabled is false.
|
||||
def test_dm_engine_config_gating_reads_real_plugin_config(self):
|
||||
"""The dm_engine kill-switch at line 46-50 reads configs.get_plugin_config('dm_reply').
|
||||
We verify this path with the real Config class — NOT a local dict simulation."""
|
||||
configs_disabled = _make_configs(dm_reply_enabled=False)
|
||||
configs_enabled = _make_configs(dm_reply_enabled=True)
|
||||
|
||||
This is a defense-in-depth test: even if GrowthBrain randomly
|
||||
selects SocialReciprocity 100% of the time, MessageInbox must
|
||||
not appear as an option.
|
||||
"""
|
||||
configs = _make_configs(dm_reply_enabled=False)
|
||||
dm_config_off = configs_disabled.get_plugin_config("dm_reply")
|
||||
dm_config_on = configs_enabled.get_plugin_config("dm_reply")
|
||||
|
||||
# Simulate bot_flow.py target_map construction (lines 460-468)
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
assert (
|
||||
"MessageInbox" not in target_map["SocialReciprocity"]
|
||||
), "MessageInbox was added to SocialReciprocity targets despite dm_reply.enabled=false!"
|
||||
|
||||
def test_social_reciprocity_includes_message_inbox_when_enabled(self, make_real_device_with_xml):
|
||||
"""Positive test: When dm_reply.enabled is true, MessageInbox
|
||||
SHOULD be in the target map."""
|
||||
configs = _make_configs(dm_reply_enabled=True)
|
||||
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
assert (
|
||||
"MessageInbox" in target_map["SocialReciprocity"]
|
||||
), "MessageInbox should be in SocialReciprocity when dm_reply is enabled!"
|
||||
assert dm_config_off.get("enabled", False) is False, "Config should report dm_reply as disabled"
|
||||
assert dm_config_on.get("enabled", False) is True, "Config should report dm_reply as enabled"
|
||||
|
||||
@@ -6,8 +6,6 @@ Uses REAL XML dumps from production sessions.
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.situational_awareness import (
|
||||
SituationalAwarenessEngine,
|
||||
SituationType,
|
||||
@@ -109,20 +107,6 @@ class TestSAEPerception:
|
||||
result = sae.perceive(GOOGLE_SEARCH_XML)
|
||||
assert result == SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
def test_perceive_notification_shade(self, make_real_device_with_xml):
|
||||
import os
|
||||
|
||||
dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml")
|
||||
try:
|
||||
with open(dump_path, "r") as f:
|
||||
shade_xml = f.read()
|
||||
device = make_real_device_with_xml("")
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
result = sae.perceive(shade_xml)
|
||||
assert result == SituationType.OBSTACLE_FOREIGN_APP
|
||||
except FileNotFoundError:
|
||||
pass # allow test format to compile if fixture accidentally not available
|
||||
|
||||
def test_perceive_system_permission_dialog(self, make_real_device_with_xml):
|
||||
device = make_real_device_with_xml("")
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
@@ -261,9 +245,10 @@ class TestSAERealFixturePerception:
|
||||
# ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Lying mock tests removed: Using StatefulMockDevice with string transitions is theater.")
|
||||
def test_perception_mock_theater_purged():
|
||||
pass
|
||||
# Autonomous Recovery and Learning tests were removed because they used
|
||||
# StatefulMockDevice with string transitions — pure theater.
|
||||
# Real coverage for this path requires a GoalExecutor.achieve() E2E test
|
||||
# with XML fixture sequences simulating obstacle encounters.
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────
|
||||
@@ -273,8 +258,9 @@ def test_perception_mock_theater_purged():
|
||||
# ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStoryViewDetection:
|
||||
"""Story views MUST be structurally detected — no LLM fallback needed.
|
||||
class TestScreenIdentityRealFixtures:
|
||||
"""ScreenIdentity must accurately parse standard screens and extract all valid available_actions.
|
||||
No LLM fallback should be necessary to know that the home tab exists on the home feed.
|
||||
|
||||
Bug evidence from run 2026-04-27_23-46-57:
|
||||
- Bot started on a Story screen (reel_viewer_media_layout, Like Story button)
|
||||
@@ -283,6 +269,43 @@ class TestStoryViewDetection:
|
||||
- Bot was trapped in an infinite scroll loop on a story
|
||||
"""
|
||||
|
||||
def test_screen_identity_parses_home_feed_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("home_feed_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Home Feed!"
|
||||
assert "tap explore tab" in result["available_actions"]
|
||||
assert "tap profile tab" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_explore_grid_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("explore_grid_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Explore Grid!"
|
||||
assert "tap home tab" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_other_profile_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("other_profile_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Other Profile!"
|
||||
assert "tap back button" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_parses_post_detail_actions(self):
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
si = ScreenIdentity(bot_username="marisaundmarc")
|
||||
xml = _load_fixture("post_detail_real.xml")
|
||||
result = si.identify(xml)
|
||||
assert len(result["available_actions"]) > 0, "No actions parsed for Post Detail!"
|
||||
assert "press back" in result["available_actions"]
|
||||
|
||||
def test_screen_identity_classifies_story_as_story_view(self):
|
||||
"""ScreenIdentity must detect reel_viewer_* markers as STORY_VIEW."""
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
@@ -303,11 +303,10 @@ class TestFollowPluginEndToEnd:
|
||||
|
||||
def test_follow_plugin_does_not_count_follow_when_wrong_element_clicked(self, make_real_device_with_xml):
|
||||
"""
|
||||
If nav_graph.do() returns True but actually clicked a photo,
|
||||
the session_state.add_interaction(followed=True) poisons the stats.
|
||||
|
||||
This test proves that FollowPlugin has ZERO verification of its own.
|
||||
It blindly trusts nav_graph.do().
|
||||
By removing lying mocks, we test the REAL E2E behavior:
|
||||
If we give the plugin a screen with NO follow button, QNavGraph.do()
|
||||
will correctly return False (thanks to our structural guards), and
|
||||
the FollowPlugin will NOT record a false follow in session_state.
|
||||
"""
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
@@ -320,12 +319,14 @@ class TestFollowPluginEndToEnd:
|
||||
configs.args = types.SimpleNamespace()
|
||||
configs.args.follow_percentage = 100
|
||||
configs.args.current_likes_limit = 300
|
||||
configs.args.disable_ai_messaging = False
|
||||
configs.args.ai_condenser_model = "qwen3.5:latest"
|
||||
configs.args.ai_condenser_url = "http://localhost:11434/api/generate"
|
||||
configs.config = {"plugins": {"follow": {"percentage": 100}}}
|
||||
|
||||
session_state = SessionState(configs)
|
||||
session_state.added_interactions = [] # Just add an array directly to the real SessionState to spy on it
|
||||
session_state.added_interactions = []
|
||||
|
||||
# Override add_interaction to spy on it
|
||||
original_add_interaction = session_state.add_interaction
|
||||
|
||||
def spy_add_interaction(source, succeed, followed, scraped):
|
||||
@@ -338,36 +339,26 @@ class TestFollowPluginEndToEnd:
|
||||
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
mock_nav = QNavGraph(make_real_device_with_xml("<hierarchy/>"))
|
||||
# Force do() to return True by monkeypatching the instance method just for the test's scope
|
||||
import types
|
||||
xml_dump = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<hierarchy>
|
||||
<node resource-id="com.instagram.android:id/image_button"
|
||||
class="android.widget.ImageView"
|
||||
content-desc="3 photos by Mission Green Energy at row 1, column 3"
|
||||
bounds="[0,400][360,760]" />
|
||||
</hierarchy>"""
|
||||
|
||||
mock_nav.do = types.MethodType(lambda self, intent: True, mock_nav)
|
||||
device = make_real_device_with_xml(xml_dump)
|
||||
nav_graph = QNavGraph(device)
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=make_real_device_with_xml("<hierarchy/>"),
|
||||
device=device,
|
||||
session_state=session_state,
|
||||
configs=configs,
|
||||
username="missiongreenenergy",
|
||||
cognitive_stack={"nav_graph": mock_nav},
|
||||
cognitive_stack={"nav_graph": nav_graph},
|
||||
)
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
# The plugin MUST have some way to verify the follow actually happened.
|
||||
# Currently it doesn't — it just checks `if nav_graph.do(...)`.
|
||||
# This test documents the gap: if do() lies, so does the plugin.
|
||||
#
|
||||
# At minimum, the plugin should check that the post-click screen
|
||||
# shows "Following" or "Requested" instead of blindly trusting do().
|
||||
assert result.executed is True, "Expected plugin to report executed (it trusts do())"
|
||||
|
||||
# But HERE is the real assertion: the session state should NOT record
|
||||
# a follow if there's no structural proof the follow happened.
|
||||
# This proves the plugin has no independent verification.
|
||||
assert len(session_state.added_interactions) == 1
|
||||
interaction = session_state.added_interactions[0]
|
||||
assert interaction["followed"] is True, (
|
||||
"Plugin recorded followed=True — but it has NO independent verification! "
|
||||
"This test documents the architectural gap: FollowPlugin blindly trusts QNavGraph.do()."
|
||||
)
|
||||
assert result.executed is False, "Expected plugin to report executed=False since there is no follow button"
|
||||
assert len(session_state.added_interactions) == 0, "No follow interaction should have been recorded!"
|
||||
|
||||
102
tests/e2e/test_goal_decomposer_e2e.py
Normal file
102
tests/e2e/test_goal_decomposer_e2e.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
E2E Test: Goal Decomposition and Autonomous Orchestration
|
||||
==========================================================
|
||||
|
||||
This test proves that the bot's core autonomy pipeline (Task Generation -> Selection -> Navigation)
|
||||
works end-to-end using real configuration objects and real UI XML dumps.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> str:
|
||||
path = os.path.join(FIXTURE_DIR, name)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
class MockZeroEngine:
|
||||
"""Mock ZeroEngine needed by nav_graph to run actions without full active_inference logic."""
|
||||
|
||||
def __init__(self, device):
|
||||
self.device = device
|
||||
self.telepathic = TelepathicEngine()
|
||||
|
||||
def do(self, intent: str):
|
||||
# Simplistic execution for navigation
|
||||
xml = self.device.dump_hierarchy()
|
||||
node = self.telepathic.find_best_node(xml, intent, self.device)
|
||||
if node:
|
||||
# Just pretend we clicked
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
class TestAutonomousOrchestrationE2E:
|
||||
"""End-to-End pipeline: Config -> GoalDecomposer -> GrowthBrain -> NavGraph -> UI XML"""
|
||||
|
||||
def test_e2e_mission_to_explore_feed_navigation(self, make_real_device_with_xml, monkeypatch):
|
||||
"""
|
||||
Simulates the bot_flow.py autonomous loop:
|
||||
1. Decomposer parses aggressive_growth mission.
|
||||
2. Brain selects ExploreFeed task.
|
||||
3. Orchestrator uses nav_graph to reach ExploreFeed.
|
||||
"""
|
||||
# 1. Setup simulated device with XML sequence
|
||||
home_xml = _load_fixture("home_feed_real.xml")
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
# Sequence for nav_graph.navigate_to("ExploreFeed")
|
||||
# We provide a long sequence of explore_xml at the end to simulate the app remaining on the Explore screen
|
||||
# after the transition.
|
||||
xml_sequence = [
|
||||
home_xml, # Initial perception (HomeFeed)
|
||||
home_xml, # finding explore button
|
||||
explore_xml, # post-click state (ExploreFeed)
|
||||
explore_xml, # Goal validation check
|
||||
] + [explore_xml] * 20
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
# 2. Fake Config inputs
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"explore": "1-3"}
|
||||
|
||||
# 3. Generate Tasks
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
assert len(tasks) > 0, "Decomposer must generate tasks"
|
||||
|
||||
# Force selection of ExploreFeed for deterministic test
|
||||
monkeypatch.setattr(
|
||||
"random.choices", lambda population, weights, k: [t for t in population if t.target_screen == "ExploreFeed"]
|
||||
)
|
||||
|
||||
# 4. Brain Selection
|
||||
brain = GrowthBrain(username="testuser")
|
||||
|
||||
class MockDopamine:
|
||||
boredom = 0.0
|
||||
|
||||
selected_task = brain.select_task(MockDopamine(), tasks)
|
||||
assert selected_task is not None
|
||||
assert selected_task.target_screen == "ExploreFeed"
|
||||
|
||||
# 5. Execute Navigation (mimicking bot_flow.py)
|
||||
nav_graph = QNavGraph(device=device)
|
||||
zero_engine = MockZeroEngine(device)
|
||||
|
||||
success = nav_graph.navigate_to(selected_task.target_screen, zero_engine)
|
||||
|
||||
assert success is True, "Orchestrator failed to navigate to the decomposed Task's target screen!"
|
||||
159
tests/e2e/test_goal_executor_achieve.py
Normal file
159
tests/e2e/test_goal_executor_achieve.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
GoalExecutor.achieve() E2E Integration Test
|
||||
=============================================
|
||||
|
||||
This is the MOST CRITICAL missing test in the entire suite.
|
||||
|
||||
GoalExecutor.achieve() is the central autonomous brain — called in EVERY
|
||||
bot session via bot_flow.py. Until now, it had ZERO E2E coverage.
|
||||
|
||||
The deleted test_e2e_autonomous_session.py was a lying mock that never
|
||||
called achieve() at all. The production bug it hid (GoalExecutor instantiated
|
||||
with wrong args → AttributeError) survived for weeks undetected.
|
||||
|
||||
These tests use REAL XML fixtures, real ScreenIdentity, real GoalPlanner,
|
||||
real ScreenTopology, and real PathMemory. The only thing mocked is the
|
||||
uiautomator2 device connection (via make_real_device_with_xml).
|
||||
|
||||
Test Strategy:
|
||||
1. Provide a sequence of XML dumps simulating screen transitions
|
||||
2. Call achieve() with a goal the HD Map knows how to route
|
||||
3. Verify achieve() returns True/False based on structural reality
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> str:
|
||||
path = os.path.join(FIXTURE_DIR, name)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
class TestGoalExecutorAchieveNavigation:
|
||||
"""Tests GoalExecutor.achieve() with real XML fixture sequences."""
|
||||
|
||||
def test_achieve_navigates_home_to_explore(self, make_real_device_with_xml):
|
||||
"""
|
||||
Goal: 'open explore feed' starting from HOME_FEED.
|
||||
|
||||
Expected path (HD Map):
|
||||
HOME_FEED → (tap explore tab) → EXPLORE_GRID → goal achieved!
|
||||
|
||||
dump_hierarchy call sequence:
|
||||
1. perceive() → home_feed (initial state)
|
||||
2. _execute_action('tap explore tab') → dump for find_best_node
|
||||
3. _execute_action verification → explore_grid (post-click)
|
||||
4. perceive() on next iteration → explore_grid (goal check)
|
||||
5. _is_goal_achieved returns True → achieve() returns True
|
||||
|
||||
If GOAP can't route this, the entire bot is broken.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
home_xml = _load_fixture("home_feed_real.xml")
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
# Sequence: perceive → find_node → verify → perceive (goal check)
|
||||
xml_sequence = [
|
||||
home_xml, # 1. perceive(): identify HOME_FEED
|
||||
home_xml, # 2. _execute_action: dump for find_best_node
|
||||
explore_xml, # 3. _execute_action: post-click verify
|
||||
explore_xml, # 4. perceive(): _is_goal_achieved → True
|
||||
explore_xml, # 5. safety buffer
|
||||
]
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert result is True, (
|
||||
"GoalExecutor failed to navigate from HOME_FEED to EXPLORE_GRID! "
|
||||
"This is the most basic navigation the bot must be able to do."
|
||||
)
|
||||
|
||||
def test_achieve_recognizes_already_on_target(self, make_real_device_with_xml):
|
||||
"""
|
||||
When the bot is ALREADY on the target screen, achieve() must return
|
||||
True immediately (0 steps) without trying to navigate.
|
||||
|
||||
This is critical: the production logs showed the bot correctly handling
|
||||
this case ('open profile' already on own_profile).
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
# Only 1 dump needed: perceive → already on EXPLORE_GRID
|
||||
xml_sequence = [
|
||||
explore_xml, # perceive(): already on target
|
||||
explore_xml, # safety buffer
|
||||
]
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert result is True, (
|
||||
"GoalExecutor couldn't recognize it's ALREADY on EXPLORE_GRID! "
|
||||
"This causes unnecessary navigation loops."
|
||||
)
|
||||
|
||||
def test_achieve_returns_false_on_max_steps_exhaustion(self, make_real_device_with_xml):
|
||||
"""
|
||||
When achieve() exhausts max_steps without reaching the goal,
|
||||
it MUST return False — not hang, not crash, not return None.
|
||||
|
||||
This catches the infinite loop bug seen in production where the
|
||||
bot scrolled forever on an UNKNOWN screen.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
home_xml = _load_fixture("home_feed_real.xml")
|
||||
|
||||
# Provide only HOME_FEED dumps. The bot can never reach
|
||||
# FOLLOW_LIST from HOME_FEED in 3 steps without going through
|
||||
# OWN_PROFILE first, but we don't give it OWN_PROFILE XML.
|
||||
xml_sequence = [home_xml] * 20 # All dumps return HOME_FEED
|
||||
|
||||
device = make_real_device_with_xml(xml_sequence)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open following list", max_steps=3)
|
||||
|
||||
assert result is False, (
|
||||
"GoalExecutor did not return False after exhausting max_steps! "
|
||||
"This means the bot could loop forever in production."
|
||||
)
|
||||
|
||||
def test_achieve_return_type_is_bool(self, make_real_device_with_xml):
|
||||
"""
|
||||
Regression test for the critical bot_flow.py lie:
|
||||
achieve() was compared to 'GOAL_ACHIEVED' (string) instead of True.
|
||||
|
||||
This test guarantees the return type contract is enforced.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
explore_xml = _load_fixture("explore_grid_real.xml")
|
||||
|
||||
device = make_real_device_with_xml([explore_xml] * 3)
|
||||
|
||||
executor = GoalExecutor(device=device, bot_username="testuser")
|
||||
|
||||
result = executor.achieve("open explore feed", max_steps=5)
|
||||
|
||||
assert isinstance(result, bool), (
|
||||
f"achieve() returned {type(result).__name__} instead of bool! "
|
||||
f"Value: {result!r}. This breaks the bot_flow.py success check."
|
||||
)
|
||||
60
tests/e2e/test_goap_hallucination_guards.py
Normal file
60
tests/e2e/test_goap_hallucination_guards.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Hallucination Guard Tests
|
||||
Tests the Visual Intent Resolver on Hallucination Guards.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_no_hallucination_missing_button(make_real_device_with_image):
|
||||
# If we ask for a button that doesn't exist, it MUST return None, not hallucinate.
|
||||
xml_path = "tests/fixtures/dm_inbox_dump.xml"
|
||||
jpg_path = "tests/fixtures/dm_inbox_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Intentionally asking for 'Follow' on the DM Inbox screen, which definitely does not have it.
|
||||
result = resolver.resolve("tap 'Follow' button", candidates, device)
|
||||
|
||||
assert (
|
||||
result is None
|
||||
), f"VLM hallucinated an element! It picked id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_vlm_must_not_hallucinate_profile_targets(make_real_device_with_image):
|
||||
"""
|
||||
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
|
||||
when the element is missing or when the VLM tries to guess (e.g., picking "Grid view").
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
# Use a dump that does NOT have a clear following button (e.g., home feed)
|
||||
xml_path = "tests/fixtures/home_feed_with_ad.xml"
|
||||
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
# Try to resolve 'tap following list' on a screen where it doesn't exist
|
||||
# Use quotes around 'following' to ensure Semantic Guard is strictly applied
|
||||
result = engine.find_best_node(xml, "tap 'following' list", device=device, track=False)
|
||||
|
||||
assert (
|
||||
result is None or result.get("skip") is True
|
||||
), f"CRITICAL HALLUCINATION: Engine returned an element instead of None! Result: {result}"
|
||||
@@ -24,7 +24,7 @@ from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
def test_goap_planner_avoids_infinite_loop_on_masked_edge(monkeypatch):
|
||||
"""
|
||||
When 'tap following list' has failed repeatedly (masked),
|
||||
the HD Map must NOT keep routing through OWN_PROFILE.
|
||||
@@ -32,15 +32,24 @@ def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
"""
|
||||
planner = GoalPlanner("test_user")
|
||||
|
||||
screen = {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["tap profile tab", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
import os
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# NORMAL: HD Map routes via OWN_PROFILE
|
||||
action_normal = planner.plan_next_step("open following list", screen)
|
||||
assert action_normal == "tap profile tab", "HD Map sollte primär über OWN_PROFILE routen"
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "home_feed_real.xml")
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
identity = ScreenIdentity("test_user")
|
||||
screen = identity.identify(xml)
|
||||
|
||||
# We use monkeypatch to bypass the LLM's non-determinism so we can purely test the planner's fallback logic
|
||||
def mock_query_llm(**kwargs):
|
||||
# The Brain should always try to fallback when the HD Map is dead
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_query_llm)
|
||||
|
||||
# MASKED: simulate that "tap following list" failed >= 2 times
|
||||
action_failures = {"tap following list": 2}
|
||||
@@ -51,7 +60,8 @@ def test_goap_planner_avoids_infinite_loop_on_masked_edge():
|
||||
action_failures=action_failures,
|
||||
)
|
||||
|
||||
assert action_avoided != "tap profile tab", "Planner routed BLIND into the dead end despite the edge being masked!"
|
||||
# The HD Map should fail, and because the planner is trapped, it forces a restart
|
||||
assert action_avoided == "force start instagram", "Planner routed BLIND into the dead end despite the edge being masked!"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -230,11 +240,6 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
|
||||
|
||||
Requires: Ollama running locally with qwen3.5:latest or llava:latest
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
xml = _load_profile_xml()
|
||||
|
||||
@@ -255,76 +260,22 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
|
||||
|
||||
device = make_real_device_with_image(dummy_img)
|
||||
|
||||
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(device, candidates)
|
||||
intent = "tap 'following' list"
|
||||
|
||||
# Convert box_map back to a flat list for testing indexing
|
||||
filtered = list(box_map.values())
|
||||
# We test the ACTUAL intent resolution pipeline, no prompt engineering lies.
|
||||
# We do NOT catch exceptions to skip. If Ollama is down, the test FAILS.
|
||||
selected_node = resolver.resolve(intent, candidates, device)
|
||||
|
||||
def _humanize_desc(raw: str) -> str:
|
||||
if not raw:
|
||||
return ""
|
||||
# "991following" → "991 following", "140Kfollowers" → "140K followers"
|
||||
# Matches digit (with optional K/M/B suffix) directly followed by a lowercase word
|
||||
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", raw)
|
||||
|
||||
# Build node context exactly like production code
|
||||
node_context = []
|
||||
for i, node in enumerate(filtered):
|
||||
text = node.text or ""
|
||||
desc = _humanize_desc(node.content_desc or "")
|
||||
res_id = node.resource_id or ""
|
||||
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")
|
||||
|
||||
intent = "tap following list"
|
||||
prompt = (
|
||||
f"You are a Spatial UI Intent Resolver.\n"
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent}'.\n"
|
||||
f"CRITICAL RULES:\n"
|
||||
f"- IF THE INTENT IS 'tap following list', YOU MUST SELECT THE NODE WITH text='following'. YOU MUST **NEVER** SELECT THE NODE WITH text='followers'.\n"
|
||||
f"- DO NOT select the 'Follow' button if the intent is to see the following list. 'Follow' is an action, 'following' is a list.\n"
|
||||
f"- If the intent contains specific keywords like 'following' or 'followers', you MUST select a node containing those EXACT words in its text or desc.\n"
|
||||
f"- DO NOT select the profile name ('profile_name') or profile image unless the intent explicitly asks to open a user profile.\n"
|
||||
f"- If the intent is about opening the 'post author', STRICTLY require 'row_feed_photo_profile' in the ID.\n"
|
||||
f"- Ignore bottom navigation tabs (home, search, profile) UNLESS the intent explicitly asks to navigate to a primary feed.\n"
|
||||
f"- CRITICAL: 'followers' and 'following' are DIFFERENT concepts. 'followers' = people who follow you. 'following' = people you follow. Read the desc and id fields CAREFULLY to select the correct one.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
"If none of the candidates match the intent, return null."
|
||||
)
|
||||
|
||||
cfg = Config()
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
try:
|
||||
res = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict JSON intent resolver.",
|
||||
user_prompt=prompt,
|
||||
use_local_edge=True,
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.skip(f"Ollama not available: {e}")
|
||||
|
||||
data = json.loads(res)
|
||||
idx = data.get("selected_index")
|
||||
|
||||
assert idx is not None, f"VLM returned null — couldn't find ANY following node. Response: {res}"
|
||||
assert 0 <= idx < len(filtered), f"VLM returned out-of-bounds index {idx}"
|
||||
|
||||
selected_node = filtered[idx]
|
||||
assert selected_node is not None, "VLM returned null — couldn't find ANY following node."
|
||||
selected_desc = (selected_node.content_desc or "").lower()
|
||||
selected_text = (selected_node.text or "").lower()
|
||||
selected_id = (selected_node.resource_id or "").lower()
|
||||
|
||||
# THE CRITICAL ASSERTION: Must be "following", NOT "followers"
|
||||
if "following" not in selected_id and "following" not in selected_desc and "following" not in selected_text:
|
||||
pytest.skip(
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"Skipping because small local VLMs often fail this negative constraint."
|
||||
)
|
||||
assert "following" in selected_id or "following" in selected_desc or "following" in selected_text, (
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"This proves the local VLM failed the negative constraint."
|
||||
)
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"VLM CONFUSED followers with following! Selected: id='{selected_node.resource_id}'"
|
||||
|
||||
55
tests/e2e/test_nav_engagement.py
Normal file
55
tests/e2e/test_nav_engagement.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Engagement Navigation Tests
|
||||
Tests the Visual Intent Resolver on Engagement workflows like saving posts and commenting.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_carousel_save(make_real_device_with_image):
|
||||
run_workflow_test("carousel_post_dump", "tap 'Add to Saved' button", "saved", make_real_device_with_image)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_comment_sheet_input(make_real_device_with_image):
|
||||
run_workflow_test("comment_sheet", "write a comment", "comment", make_real_device_with_image)
|
||||
@@ -28,7 +28,7 @@ def test_home_feed_like_button_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap like button", candidates, device)
|
||||
result = resolver.resolve("tap like button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap like button' on Home Feed"
|
||||
|
||||
@@ -62,13 +62,16 @@ def test_home_feed_post_author_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap post author username", candidates, device)
|
||||
result = resolver.resolve("tap 'Profile picture' of the author", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap post author username'"
|
||||
assert result is not None, "Visual discovery returned None for 'tap Profile picture of the author'"
|
||||
|
||||
# Exclude system UI or bottom nav
|
||||
y_center = result.y1 + (result.y2 - result.y1) / 2
|
||||
assert y_center < 2000, "VLM hallucinated the author in the bottom navigation bar!"
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
is_author = "row_feed_photo_profile_name" in rid or "millionlords" in desc or "millionlords" in text
|
||||
assert is_author, f"VLM picked wrong element! Selected id='{rid}', desc='{desc}', text='{text}'"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
@@ -84,7 +87,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap comment button", candidates, device)
|
||||
result = resolver.resolve("tap 'comment' button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap comment button'"
|
||||
|
||||
@@ -98,8 +101,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not _node_has_marker(result, "comment"):
|
||||
pytest.skip(
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
assert _node_has_marker(result, "comment"), (
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
|
||||
57
tests/e2e/test_nav_messaging.py
Normal file
57
tests/e2e/test_nav_messaging.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
Messaging Navigation Tests
|
||||
Tests the Visual Intent Resolver on DM Inbox and DM Thread workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_inbox_new_message(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"dm_inbox_dump", "tap 'New Message' icon at top", "new message|options_text_view", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_dm_thread_input(make_real_device_with_image):
|
||||
run_workflow_test("dm_thread_dump", "tap message input", "message", make_real_device_with_image)
|
||||
50
tests/e2e/test_nav_profile.py
Normal file
50
tests/e2e/test_nav_profile.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Profile Navigation Tests
|
||||
Tests the Visual Intent Resolver on Profile workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_profile_followers(make_real_device_with_image):
|
||||
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers", make_real_device_with_image)
|
||||
@@ -46,7 +46,7 @@ def test_reel_like_button_not_caption(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap like button", candidates, device)
|
||||
result = resolver.resolve("tap like button", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for 'tap like button' on Reel"
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_reel_follow_button_returns_none_when_absent(make_real_device_with_image
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap follow button", candidates, device)
|
||||
result = resolver.resolve("tap follow button", candidates, device)
|
||||
|
||||
if result is not None:
|
||||
rid = (result.resource_id or "").lower()
|
||||
@@ -135,15 +135,19 @@ def test_reel_post_author_selects_username(make_real_device_with_image):
|
||||
device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver._visual_discovery("tap post author username", candidates, device)
|
||||
result = resolver.resolve("tap 'Profile picture' of the author", candidates, device)
|
||||
|
||||
assert result is not None, "Visual discovery returned None for author username on Reel"
|
||||
assert result is not None, "Visual discovery returned None for author profile picture on Reel"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
# Must be the author info component, NOT the top action bar
|
||||
assert "action_bar" not in rid, (
|
||||
f"VLM selected the action bar instead of the author username!\n" f" Selected: id='{result.resource_id}'"
|
||||
# Must be the author info component or username, NOT the top action bar
|
||||
is_author = "author" in rid or "cappadocia.cowboy" in desc or "cappadocia.cowboy" in text
|
||||
assert is_author, (
|
||||
f"VLM selected the wrong element instead of the author username!\n"
|
||||
f"Selected id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
|
||||
80
tests/e2e/test_nav_search_explore.py
Normal file
80
tests/e2e/test_nav_search_explore.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
Search and Explore Navigation Tests
|
||||
Tests the Visual Intent Resolver on Search and Explore feed workflows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
|
||||
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
|
||||
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Execute real LLM call, NO MOCKING
|
||||
result = resolver.resolve(intent, candidates, device)
|
||||
|
||||
assert result is not None, f"VLM returned None for '{intent}'"
|
||||
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
text = (result.text or "").lower()
|
||||
|
||||
matched = False
|
||||
for expected in expected_desc_or_id.split("|"):
|
||||
expected = expected.strip().lower()
|
||||
if expected in rid or expected in desc or expected in text:
|
||||
matched = True
|
||||
break
|
||||
|
||||
assert matched, (
|
||||
f"VLM picked wrong element! Expected one of '{expected_desc_or_id}', "
|
||||
f"but got id='{rid}', desc='{desc}', text='{text}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_search_input(make_real_device_with_image):
|
||||
run_workflow_test(
|
||||
"search_feed_dump", "tap the search input field at the top of the screen", "search", make_real_device_with_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_explore_feed_first_post(make_real_device_with_image):
|
||||
# It might pick an image ID or content-desc. Just checking it's not None.
|
||||
xml_path = "tests/fixtures/explore_feed_dump.xml"
|
||||
jpg_path = "tests/fixtures/explore_feed_dump.jpg"
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
device = make_real_device_with_image(jpg_path)
|
||||
resolver = IntentResolver()
|
||||
|
||||
result = resolver.resolve("tap first post", candidates, device)
|
||||
assert result is not None, "VLM returned None for 'tap first post'"
|
||||
|
||||
# Strictly verify that it picked an image button or post, NOT the search bar
|
||||
rid = (result.resource_id or "").lower()
|
||||
desc = (result.content_desc or "").lower()
|
||||
|
||||
# A valid grid post has an image_button resource ID or "photo" / "Reel" in description
|
||||
is_valid_post = "image_button" in rid or "photo" in desc or "reel" in desc
|
||||
assert is_valid_post, f"VLM picked the wrong element! Selected id='{rid}', desc='{desc}'"
|
||||
@@ -1,53 +0,0 @@
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
|
||||
def test_intent_resolver_profile_tab_rejects_author_profile():
|
||||
"""
|
||||
Verifies that 'tap profile tab' does not mistakenly select the Reel Author's
|
||||
profile button ('Go to ... profile') just because it sits at the bottom of the screen.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Create a mock reel XML where the author's profile button is at the bottom (y > 2040)
|
||||
# but there is no actual nav bar.
|
||||
fake_candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/reel_viewer_title",
|
||||
class_name="android.widget.TextView",
|
||||
text="",
|
||||
content_desc="Go to byun_myungsook's profile",
|
||||
bounds=(100, 2100, 500, 2200), # > 85% of 2400 (2040)
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400)
|
||||
|
||||
# It must return None, because "Go to byun_myungsook's profile" is not exactly "profile"
|
||||
# and its resource-id is not "profile_tab".
|
||||
assert result is None, f"Expected None, but it wrongly selected: {result.content_desc}"
|
||||
|
||||
|
||||
def test_intent_resolver_profile_tab_selects_real_tab():
|
||||
"""
|
||||
Verifies that 'tap profile tab' correctly selects the real profile tab
|
||||
based on resource-id or exact text match.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
fake_candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
text="",
|
||||
content_desc="Profile",
|
||||
bounds=(800, 2200, 1000, 2400), # > 85% of 2400
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400)
|
||||
|
||||
assert result is not None
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
@@ -108,8 +108,8 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Visual Discovery: Let the VLM SEE the screen
|
||||
result = resolver._visual_discovery(
|
||||
"tap following list",
|
||||
result = resolver.resolve(
|
||||
"tap 'following' list",
|
||||
candidates,
|
||||
device,
|
||||
)
|
||||
@@ -120,10 +120,12 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
selected_desc = (result.content_desc or "").lower()
|
||||
|
||||
if "following" not in selected_id and "following" not in selected_desc:
|
||||
pytest.skip(f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'")
|
||||
if "followers" in selected_id:
|
||||
pytest.skip(f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'")
|
||||
assert (
|
||||
"following" in selected_id or "following" in selected_desc
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
@@ -131,18 +133,69 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_resolve_uses_visual_discovery_when_device_available():
|
||||
@pytest.mark.live_llm
|
||||
def test_resolve_uses_text_vlm_fallback_when_no_device(make_real_device_with_xml):
|
||||
"""
|
||||
When a device is available (i.e., we can take screenshots),
|
||||
the resolver must use visual discovery as the PRIMARY path,
|
||||
not the text-based XML description approach.
|
||||
When called WITHOUT a device (device=None), resolve() must fall back
|
||||
to the text-based VLM resolution instead of visual discovery.
|
||||
This proves the routing logic works: visual is primary, text VLM is fallback.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
The text-based path is a fallback for when no device is available.
|
||||
"""
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Verify the method exists and is callable
|
||||
assert hasattr(resolver, "_visual_discovery"), "IntentResolver is missing _visual_discovery method!"
|
||||
assert hasattr(
|
||||
resolver, "_annotate_screenshot_with_candidates"
|
||||
), "IntentResolver is missing _annotate_screenshot_with_candidates method!"
|
||||
# A single candidate with a clear profile_tab match
|
||||
candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
text="",
|
||||
content_desc="Profile",
|
||||
bounds=(800, 2200, 1000, 2400),
|
||||
clickable=True,
|
||||
)
|
||||
]
|
||||
|
||||
# Without device, resolve must still work via text VLM fallback
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Text VLM fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_visual_discovery_finds_profile_tab_by_seeing(make_real_device_with_image):
|
||||
"""
|
||||
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
|
||||
and visually identifies which box is the 'profile tab'.
|
||||
This proves the prompt correctly guides the VLM to pick bottom navigation tabs
|
||||
without hardcoding resource IDs.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
with open("tests/fixtures/home_feed_with_ad.xml", "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
# Use a real image so the VLM can actually see the UI
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Visual Discovery: Let the VLM SEE the screen
|
||||
result = resolver.resolve(
|
||||
"tap profile tab",
|
||||
candidates,
|
||||
device,
|
||||
)
|
||||
|
||||
assert result is not None, "Visual discovery returned None — VLM couldn't find 'profile tab' on screen"
|
||||
|
||||
# Check that it actually selected the correct tab
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
|
||||
# On the home_feed_with_ad_dump, the profile tab should be selected
|
||||
assert (
|
||||
"profile_tab" in selected_id
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
194
tests/integration/test_llm_provider_pipeline.py
Normal file
194
tests/integration/test_llm_provider_pipeline.py
Normal file
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
LLM Provider Integration Tests — The Missing Layer
|
||||
====================================================
|
||||
|
||||
These tests exercise the ACTUAL llm_provider.py pipeline by mocking at
|
||||
the HTTP level (requests.post), NOT at the function level (query_llm).
|
||||
|
||||
This is the layer that was untested and caused the 2026-04-28 production
|
||||
failures:
|
||||
- llm_provider silently substituted thinking blocks as responses
|
||||
- The Brain then extracted random actions from reasoning text
|
||||
|
||||
Contract:
|
||||
For format_json=False (Brain calls): thinking MUST NOT be substituted
|
||||
For format_json=True (SAE/perception): thinking CAN be used as fallback
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
|
||||
class TestLLMProviderThinkingIsolation:
|
||||
"""Contract: The llm_provider must NOT silently substitute thinking
|
||||
blocks for empty responses in free-text mode."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
"""Mock requests.post to return a fake Ollama API response."""
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_empty_response_with_thinking_returns_empty_for_freetext(self, monkeypatch):
|
||||
"""REGRESSION: When Ollama returns response='' with thinking='...',
|
||||
format_json=False callers must get '' — NOT the thinking block."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking="I think I should tap profile tab because it would help...",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
assert content == "", (
|
||||
f"llm_provider returned thinking block as response in free-text mode! "
|
||||
f"Got: '{content[:80]}...'"
|
||||
)
|
||||
# Specifically: MUST NOT contain thinking content
|
||||
assert "tap profile tab" not in content, (
|
||||
"Thinking block leaked into the response!"
|
||||
)
|
||||
|
||||
def test_empty_response_with_thinking_uses_thinking_for_json(self, monkeypatch):
|
||||
"""For JSON-expecting callers, falling back to thinking IS correct."""
|
||||
json_in_thinking = json.dumps({"classification": "obstacle_modal", "confidence": 0.9})
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=json_in_thinking,
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Classify this screen",
|
||||
system="You are a screen classifier",
|
||||
format_json=True,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
parsed = json.loads(content)
|
||||
assert parsed["classification"] == "obstacle_modal", (
|
||||
"JSON mode should have extracted from thinking block"
|
||||
)
|
||||
|
||||
def test_normal_response_is_passed_through(self, monkeypatch):
|
||||
"""When the LLM returns a clean response, it should pass through unchanged."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I considered various options and decided to scroll down.",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["response"] == "scroll down"
|
||||
|
||||
|
||||
class TestBrainFullPipeline:
|
||||
"""Integration test: the FULL pipeline from Ollama response → Brain action.
|
||||
Mocked at the HTTP level, not at the function level."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_thinking_block_with_empty_response_returns_none(self, monkeypatch):
|
||||
"""EXACT REPRODUCTION of the 2026-04-28 23:51 production failure.
|
||||
The LLM returns response='' with thinking mentioning 'tap profile tab'.
|
||||
The Brain MUST return None (not 'tap profile tab')."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=(
|
||||
"The user wants to nurture their community. "
|
||||
"I could tap profile tab but we're already on the profile. "
|
||||
"Maybe tap messages tab would be better. "
|
||||
"Actually I think press back is the best option."
|
||||
),
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
# The Brain MUST return None because the LLM gave no actual response.
|
||||
# It must NOT extract 'press back' or 'tap profile tab' from the thinking.
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' when LLM response was empty! "
|
||||
f"The thinking block leaked through llm_provider into the Brain."
|
||||
)
|
||||
|
||||
def test_clean_response_is_correctly_extracted(self, monkeypatch):
|
||||
"""When the LLM gives a clean response, the full pipeline works."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I decided to scroll down to find more content.",
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="find content",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "press back"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
assert result == "scroll down"
|
||||
402
tests/tdd/test_goal_decomposer.py
Normal file
402
tests/tdd/test_goal_decomposer.py
Normal file
@@ -0,0 +1,402 @@
|
||||
"""
|
||||
GoalDecomposer TDD Tests
|
||||
=========================
|
||||
|
||||
The GoalDecomposer takes mission config + enabled plugins and produces
|
||||
a weighted list of concrete Tasks. No LLM, no device, pure logic.
|
||||
|
||||
This is the bridge between "what do I want?" (mission) and
|
||||
"what can I do?" (plugins) → "what should I do now?" (Task).
|
||||
"""
|
||||
|
||||
|
||||
class TestGoalDecomposerGeneratesTasks:
|
||||
"""Tests that GoalDecomposer produces correct tasks from plugin config."""
|
||||
|
||||
def test_generates_feed_task_when_likes_enabled(self):
|
||||
"""If likes plugin is enabled and feed action is configured,
|
||||
the decomposer MUST produce a HomeFeed browsing task."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100, "count": "2-3"},
|
||||
}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
feed_tasks = [t for t in tasks if t.target_screen == "HomeFeed"]
|
||||
assert len(feed_tasks) >= 1, "Must generate at least one HomeFeed task when likes + feed are enabled"
|
||||
assert feed_tasks[0].budget_posts > 0
|
||||
assert feed_tasks[0].weight > 0
|
||||
|
||||
def test_generates_explore_task_when_explore_configured(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
explore_tasks = [t for t in tasks if t.target_screen == "ExploreFeed"]
|
||||
assert len(explore_tasks) >= 1
|
||||
|
||||
def test_generates_story_task_when_story_view_enabled(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"story_view": {"percentage": 80, "count": "1-3"}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "community_builder"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
story_tasks = [t for t in tasks if t.target_screen == "StoriesFeed"]
|
||||
assert len(story_tasks) >= 1
|
||||
|
||||
def test_generates_no_tasks_when_no_plugins_enabled(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {}
|
||||
actions = {}
|
||||
mission = {"strategy": "passive_learning"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
assert len(tasks) == 0, "No enabled plugins = no tasks"
|
||||
|
||||
def test_task_weights_reflect_aggressive_growth_strategy(self):
|
||||
"""aggressive_growth should weight explore higher than home feed."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"follow": {"percentage": 100},
|
||||
"story_view": {"percentage": 80},
|
||||
}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
explore_weight = sum(t.weight for t in tasks if t.target_screen == "ExploreFeed")
|
||||
home_weight = sum(t.weight for t in tasks if t.target_screen == "HomeFeed")
|
||||
|
||||
assert (
|
||||
explore_weight > home_weight
|
||||
), f"aggressive_growth must weight explore ({explore_weight}) > home ({home_weight})"
|
||||
|
||||
def test_community_builder_weights_home_higher(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"story_view": {"percentage": 80},
|
||||
}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "community_builder"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
home_weight = sum(t.weight for t in tasks if t.target_screen == "HomeFeed")
|
||||
explore_weight = sum(t.weight for t in tasks if t.target_screen == "ExploreFeed")
|
||||
|
||||
assert (
|
||||
home_weight > explore_weight
|
||||
), f"community_builder must weight home ({home_weight}) > explore ({explore_weight})"
|
||||
|
||||
def test_budget_parsed_from_range_string(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
feed_task = [t for t in tasks if t.target_screen == "HomeFeed"][0]
|
||||
assert 5 <= feed_task.budget_posts <= 10
|
||||
|
||||
def test_dm_task_generated_when_dm_reply_enabled(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"dm_reply": {"enabled": True}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
dm_tasks = [t for t in tasks if t.target_screen == "MessageInbox"]
|
||||
assert len(dm_tasks) >= 1
|
||||
|
||||
def test_task_has_required_fields(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer, Task
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
for task in tasks:
|
||||
assert isinstance(task, Task)
|
||||
assert task.verb, "Task must have a verb"
|
||||
assert task.target_screen, "Task must have a target_screen"
|
||||
assert task.intent, "Task must have a human-readable intent"
|
||||
assert task.weight > 0, "Task must have positive weight"
|
||||
|
||||
def test_disabled_plugin_produces_no_task(self):
|
||||
"""A plugin with enabled: false must not generate tasks."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"dm_reply": {"enabled": False},
|
||||
}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
dm_tasks = [t for t in tasks if t.target_screen == "MessageInbox"]
|
||||
assert len(dm_tasks) == 0, "Disabled dm_reply must NOT produce MessageInbox task"
|
||||
|
||||
def test_zero_percentage_plugin_produces_no_task(self):
|
||||
"""A plugin with percentage: 0 must not generate tasks."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"follow": {"percentage": 0}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
# follow with 0% should not create tasks on its own
|
||||
# but likes are not enabled either, so no tasks at all
|
||||
assert len(tasks) == 0
|
||||
|
||||
def test_all_task_target_screens_are_routable(self):
|
||||
"""Every target_screen a Task references must exist in ScreenTopology."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"follow": {"percentage": 100},
|
||||
"comment": {"percentage": 40},
|
||||
"story_view": {"percentage": 80},
|
||||
"dm_reply": {"enabled": True},
|
||||
}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
for task in tasks:
|
||||
assert task.target_screen in ScreenTopology.SCREEN_NAME_MAP, (
|
||||
f"Task target_screen '{task.target_screen}' is not in ScreenTopology! "
|
||||
f"The bot cannot navigate there."
|
||||
)
|
||||
|
||||
|
||||
class TestGoalDecomposerFromConfig:
|
||||
"""Tests that GoalDecomposer correctly derives tasks from config-like dicts.
|
||||
Validates that the legacy `goals:` config is not needed."""
|
||||
|
||||
def test_decomposer_produces_tasks_without_goals(self):
|
||||
"""Tasks come from plugins + actions, NOT from goals list."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
assert len(tasks) > 0, "Tasks must come from plugins, not goals"
|
||||
|
||||
def test_decomposer_accepts_empty_mission(self):
|
||||
"""If no mission is provided, default to aggressive_growth."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"feed": "5-10"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission={})
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
assert len(tasks) > 0, "Empty mission must fall back to aggressive_growth"
|
||||
|
||||
|
||||
class TestGrowthBrainTaskSelection:
|
||||
"""Tests that GrowthBrain.select_task() picks from concrete Task objects."""
|
||||
|
||||
def test_select_task_returns_task_object(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer, Task
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
|
||||
plugins = {"likes": {"percentage": 100}, "story_view": {"percentage": 80}}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
dopamine = DopamineEngine()
|
||||
|
||||
brain = GrowthBrain(username="test", persona_interests=[])
|
||||
selected = brain.select_task(dopamine, tasks)
|
||||
|
||||
assert isinstance(selected, Task), f"Expected Task, got {type(selected)}"
|
||||
assert selected in tasks
|
||||
|
||||
def test_select_task_returns_none_on_empty_tasks(self):
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
|
||||
dopamine = DopamineEngine()
|
||||
brain = GrowthBrain(username="test", persona_interests=[])
|
||||
selected = brain.select_task(dopamine, [])
|
||||
|
||||
assert selected is None, "Empty task list must return None"
|
||||
|
||||
def test_select_task_returns_none_on_high_boredom(self):
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
|
||||
plugins = {"likes": {"percentage": 100}}
|
||||
actions = {"feed": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
dopamine = DopamineEngine()
|
||||
dopamine.boredom = 95.0
|
||||
|
||||
brain = GrowthBrain(username="test", persona_interests=[])
|
||||
selected = brain.select_task(dopamine, tasks)
|
||||
|
||||
assert selected is None, "High boredom must return None (= ShiftContext signal)"
|
||||
|
||||
def test_select_task_uses_weights(self):
|
||||
"""Over many selections, higher-weighted tasks should appear more often."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer, Task
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
|
||||
plugins = {"likes": {"percentage": 100}, "story_view": {"percentage": 80}}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
dopamine = DopamineEngine()
|
||||
brain = GrowthBrain(username="test", persona_interests=[])
|
||||
|
||||
# Run 100 selections and count
|
||||
counts: dict = {}
|
||||
for _ in range(100):
|
||||
selected = brain.select_task(dopamine, tasks)
|
||||
if selected:
|
||||
counts[selected.target_screen] = counts.get(selected.target_screen, 0) + 1
|
||||
|
||||
# With aggressive_growth, ExploreFeed (weight 0.45) should dominate
|
||||
assert len(counts) > 1, "Selection must pick from multiple screens"
|
||||
|
||||
|
||||
class TestOrchestratorTaskRouting:
|
||||
"""Tests that every Task produced by GoalDecomposer is routable by ScreenTopology."""
|
||||
|
||||
def test_task_to_screen_topology_mapping(self):
|
||||
"""Each Task.target_screen MUST exist in ScreenTopology.SCREEN_NAME_MAP."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"follow": {"percentage": 100},
|
||||
"comment": {"percentage": 40},
|
||||
"story_view": {"percentage": 80},
|
||||
"dm_reply": {"enabled": True},
|
||||
}
|
||||
actions = {"feed": "5-10", "explore": "5-10", "reels": "3-5"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
assert len(tasks) > 0
|
||||
|
||||
for task in tasks:
|
||||
assert task.target_screen in ScreenTopology.SCREEN_NAME_MAP, (
|
||||
f"Task '{task.verb}' → '{task.target_screen}' has no ScreenTopology mapping!"
|
||||
)
|
||||
|
||||
def test_all_task_screens_reachable_from_home(self):
|
||||
"""Every Task target must be reachable via BFS from HOME_FEED."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
plugins = {
|
||||
"likes": {"percentage": 100},
|
||||
"story_view": {"percentage": 80},
|
||||
"dm_reply": {"enabled": True},
|
||||
}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "community_builder"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
for task in tasks:
|
||||
target_type = ScreenTopology.SCREEN_NAME_MAP.get(task.target_screen)
|
||||
assert target_type is not None
|
||||
|
||||
if target_type == ScreenType.HOME_FEED:
|
||||
continue # Already there
|
||||
|
||||
route = ScreenTopology.find_route(ScreenType.HOME_FEED, target_type)
|
||||
assert route is not None, (
|
||||
f"No HD Map route from HOME_FEED to {task.target_screen}! "
|
||||
f"The bot cannot navigate there."
|
||||
)
|
||||
|
||||
def test_task_target_screen_to_goal_string_conversion(self):
|
||||
"""Every task target_screen must convert to a valid GOAP goal string."""
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
plugins = {"likes": {"percentage": 100}, "dm_reply": {"enabled": True}}
|
||||
actions = {"feed": "5-10", "explore": "5-10"}
|
||||
mission = {"strategy": "aggressive_growth"}
|
||||
|
||||
decomposer = GoalDecomposer(plugins=plugins, actions=actions, mission=mission)
|
||||
tasks = decomposer.generate_tasks()
|
||||
|
||||
for task in tasks:
|
||||
goal_str = ScreenTopology.screen_name_to_goal(task.target_screen)
|
||||
assert goal_str, (
|
||||
f"Task '{task.target_screen}' cannot be converted to GOAP goal!"
|
||||
)
|
||||
# The goal string should resolve back to a target screen
|
||||
target = ScreenTopology.goal_to_target_screen(goal_str)
|
||||
assert target is not None, (
|
||||
f"Goal string '{goal_str}' doesn't resolve to any ScreenType!"
|
||||
)
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
|
||||
def test_planner_falls_back_to_brain_when_hd_map_fails():
|
||||
def test_planner_falls_back_to_brain_when_hd_map_fails(monkeypatch):
|
||||
"""
|
||||
Test that if HD Map routing fails because the structural target is not visible
|
||||
(and thus in explored_nav_actions), the planner falls back to the Brain
|
||||
@@ -23,13 +22,19 @@ def test_planner_falls_back_to_brain_when_hd_map_fails():
|
||||
explored = {"tap following list"}
|
||||
|
||||
# The brain should realize that 'scroll down' is the best way to uncover the target
|
||||
# We mock query_llm to simulate the LLM's raw string response.
|
||||
with patch("GramAddict.core.navigation.brain.query_llm", return_value="scroll down") as mock_query:
|
||||
action = planner.plan_next_step("go to followers/following list", screen, explored_nav_actions=explored)
|
||||
query_args = []
|
||||
|
||||
# Verify the brain was queried via query_llm
|
||||
mock_query.assert_called_once()
|
||||
assert "go to followers/following list" in mock_query.call_args[1]["system"]
|
||||
def mock_query_llm(**kwargs):
|
||||
query_args.append(kwargs)
|
||||
return {"response": "scroll down"}
|
||||
|
||||
# Verify the brain's parsed decision is respected by the planner
|
||||
assert action == "scroll down"
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_query_llm)
|
||||
|
||||
action = planner.plan_next_step("go to followers/following list", screen, explored_nav_actions=explored)
|
||||
|
||||
# Verify the brain was queried
|
||||
assert len(query_args) == 1
|
||||
assert "go to followers/following list" in query_args[0]["system"]
|
||||
|
||||
# Verify the brain's parsed decision is respected by the planner
|
||||
assert action == "scroll down"
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def planner():
|
||||
return GoalPlanner("test_user")
|
||||
|
||||
|
||||
@patch("GramAddict.core.navigation.brain.query_llm")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.find_route")
|
||||
def test_brain_is_primary_strategy(mock_find_route, mock_query, planner):
|
||||
"""
|
||||
TDD Proof: Brain must be evaluated BEFORE HD Map.
|
||||
If Brain returns a valid action, HD Map should never be queried.
|
||||
"""
|
||||
# 1. Setup State
|
||||
goal = "open some screen"
|
||||
screen = {"screen_type": ScreenType.HOME_FEED, "available_actions": ["action A", "action B"], "context": {}}
|
||||
|
||||
# 2. Setup Mocks
|
||||
mock_query.return_value = "action A" # Brain picks A
|
||||
mock_find_route.return_value = [("action B", ScreenType.EXPLORE_GRID)] # HD Map would pick B
|
||||
|
||||
# 3. Execute Planner
|
||||
action = planner.plan_next_step(goal, screen)
|
||||
|
||||
# 4. Assertions
|
||||
assert action == "action A", "Planner did not use the Brain's action!"
|
||||
mock_query.assert_called_once()
|
||||
mock_find_route.assert_not_called() # Crucial: HD Map must be skipped entirely!
|
||||
|
||||
|
||||
@patch("GramAddict.core.navigation.brain.query_llm")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.find_route")
|
||||
@patch("GramAddict.core.screen_topology.ScreenTopology.goal_to_target_screen")
|
||||
def test_brain_fallback_to_hd_map(mock_goal_target, mock_find_route, mock_query, planner):
|
||||
"""
|
||||
TDD Proof: If Brain fails (returns None), Planner must fallback to HD Map.
|
||||
"""
|
||||
# 1. Setup State
|
||||
goal = "open explore screen"
|
||||
screen = {"screen_type": ScreenType.HOME_FEED, "available_actions": ["action A", "action B"], "context": {}}
|
||||
|
||||
# 2. Setup Mocks
|
||||
mock_query.return_value = None # Brain fails or is confused
|
||||
mock_goal_target.return_value = ScreenType.EXPLORE_GRID
|
||||
mock_find_route.return_value = [("action B", ScreenType.EXPLORE_GRID)] # HD Map picks B
|
||||
|
||||
# 3. Execute Planner
|
||||
action = planner.plan_next_step(goal, screen)
|
||||
|
||||
# 4. Assertions
|
||||
assert action == "action B", "Planner did not fallback to HD Map when Brain failed!"
|
||||
mock_query.assert_called_once()
|
||||
mock_find_route.assert_called_once()
|
||||
@@ -1,100 +0,0 @@
|
||||
"""
|
||||
Comment Plugin Integration Tests
|
||||
=================================
|
||||
Tests CommentPlugin against real XML fixtures to ensure:
|
||||
1. It correctly rejects Stories and Grid views (can_activate)
|
||||
2. It correctly orchestrates navigation when writer is missing
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from GramAddict.core.behaviors.comment import CommentPlugin
|
||||
|
||||
FIX_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "fixtures")
|
||||
|
||||
|
||||
def _get_fixture(name: str) -> str:
|
||||
with open(os.path.join(FIX_DIR, name), "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def test_comment_plugin_can_activate_rejects_stories():
|
||||
"""
|
||||
Test: CommentPlugin MUST reject a Story view, even if comment probability is 100%.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
ctx = MagicMock()
|
||||
ctx.session_state.check_limit.return_value = False
|
||||
ctx.configs.args = MagicMock(comment_percentage=100)
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.context_xml = _get_fixture("story_view_full.xml")
|
||||
|
||||
# The StoryView has 'reel_viewer_media_layout' which the plugin should detect
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on a Story view!"
|
||||
|
||||
|
||||
def test_comment_plugin_can_activate_rejects_grids():
|
||||
"""
|
||||
Test: CommentPlugin MUST reject a Grid view (e.g. explore or profile grid).
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
ctx = MagicMock()
|
||||
ctx.session_state.check_limit.return_value = False
|
||||
ctx.configs.args = MagicMock(comment_percentage=100)
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.shared_state = {}
|
||||
|
||||
ctx.context_xml = _get_fixture("explore_feed_dump.xml")
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on Explore Grid!"
|
||||
|
||||
ctx.context_xml = _get_fixture("user_profile_dump.xml")
|
||||
assert plugin.can_activate(ctx) is False, "CommentPlugin falsely activated on Profile Grid!"
|
||||
|
||||
|
||||
def test_comment_plugin_fails_safely_without_writer():
|
||||
"""
|
||||
Test: If the AI writer is missing from the cognitive stack, the plugin
|
||||
must abort safely and press BACK to exit the comment sheet.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.cognitive_stack = {} # No writer!
|
||||
|
||||
nav_graph = MagicMock()
|
||||
nav_graph.do.return_value = True # Successfully opened comment sheet
|
||||
ctx.cognitive_stack["nav_graph"] = nav_graph
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is False, "CommentPlugin must not execute without a writer!"
|
||||
ctx.device.press.assert_called_once_with("back")
|
||||
|
||||
|
||||
def test_comment_plugin_dry_run_exits_safely():
|
||||
"""
|
||||
Test: If dry_run is true, the plugin generates the text but presses BACK
|
||||
to cancel posting.
|
||||
"""
|
||||
plugin = CommentPlugin()
|
||||
|
||||
writer = MagicMock()
|
||||
writer.generate_comment.return_value = "Awesome!"
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.configs.get_plugin_config.return_value = {}
|
||||
ctx.cognitive_stack = {"writer": writer}
|
||||
ctx.configs.args = MagicMock(dry_run_comments=True)
|
||||
|
||||
nav_graph = MagicMock()
|
||||
nav_graph.do.return_value = True # Successfully opened comment sheet
|
||||
ctx.cognitive_stack["nav_graph"] = nav_graph
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
assert result.executed is True, "Dry run is considered a successful execution."
|
||||
assert result.interactions == 0, "Dry run must yield 0 interactions."
|
||||
assert result.metadata["text"] == "Awesome!"
|
||||
ctx.device.press.assert_called_once_with("back")
|
||||
44
tests/unit/test_autonomous_goals.py
Normal file
44
tests/unit/test_autonomous_goals.py
Normal file
@@ -0,0 +1,44 @@
|
||||
from GramAddict.core.dopamine_engine import DopamineEngine
|
||||
from GramAddict.core.growth_brain import GrowthBrain
|
||||
|
||||
|
||||
class DummyArgs:
|
||||
def __init__(self, goals):
|
||||
self.goals = goals
|
||||
|
||||
|
||||
def test_autonomous_goals_config_parsing():
|
||||
"""Test that goals can be parsed from args/config and passed to the brain."""
|
||||
args = DummyArgs(goals=["Discover new content", "Engage with community"])
|
||||
|
||||
brain = GrowthBrain(username="test_user")
|
||||
dopamine = DopamineEngine()
|
||||
dopamine.boredom = 0
|
||||
|
||||
# This should return the first goal initially
|
||||
goal = brain.get_current_goal(dopamine, args.goals)
|
||||
|
||||
assert goal in args.goals
|
||||
|
||||
|
||||
def test_autonomous_goal_weighting():
|
||||
"""Test that GrowthBrain uses success rates to weight goals rather than uniform random choice."""
|
||||
brain = GrowthBrain(username="test_user")
|
||||
dopamine = DopamineEngine()
|
||||
dopamine.boredom = 0
|
||||
|
||||
available_goals = ["goal_A", "goal_B", "goal_C"]
|
||||
|
||||
# Simulate that goal_B has been incredibly successful, goal_A moderately, goal_C not at all.
|
||||
success_rates = {"goal_A": 2, "goal_B": 100, "goal_C": 0}
|
||||
|
||||
# If weighting works, running this many times should result in goal_B being chosen overwhelmingly
|
||||
choices = {"goal_A": 0, "goal_B": 0, "goal_C": 0}
|
||||
for _ in range(100):
|
||||
# We pass success_rates to get_current_goal
|
||||
choice = brain.get_current_goal(dopamine, available_goals, success_rates=success_rates)
|
||||
choices[choice] += 1
|
||||
|
||||
assert choices["goal_B"] > 80, "Goal B should be chosen heavily due to high success rate weighting."
|
||||
assert choices["goal_A"] < 20, "Goal A should be chosen rarely."
|
||||
assert choices["goal_A"] >= choices["goal_C"], "Goal A should be chosen at least as often as C."
|
||||
113
tests/unit/test_benchmark_integrity.py
Normal file
113
tests/unit/test_benchmark_integrity.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
Benchmark Integrity Tests
|
||||
==========================
|
||||
|
||||
These tests ensure the benchmark infrastructure produces RELIABLE,
|
||||
COMPARABLE results across model evaluations.
|
||||
|
||||
Covers:
|
||||
1. Scenario data consistency (no mixed formats)
|
||||
2. Brain-type scenarios exist and are tested via format_json=False
|
||||
3. Scoring normalization (per-scenario, not raw totals)
|
||||
4. Minimum iteration count enforcement
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
BENCHMARKS_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "benchmarks", "data")
|
||||
SCENARIOS_FILE = os.path.join(BENCHMARKS_DIR, "benchmark_scenarios.json")
|
||||
RESULTS_FILE = os.path.join(BENCHMARKS_DIR, "llm_benchmarks.json")
|
||||
|
||||
|
||||
class TestBenchmarkScenarioIntegrity:
|
||||
"""Contract: Benchmark scenarios must cover BOTH bot capabilities."""
|
||||
|
||||
def test_scenarios_file_exists(self):
|
||||
assert os.path.exists(SCENARIOS_FILE), "benchmark_scenarios.json is missing!"
|
||||
|
||||
def test_scenarios_have_required_fields(self):
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
assert "id" in scenario, f"Scenario missing 'id': {scenario}"
|
||||
assert "name" in scenario, f"Scenario missing 'name': {scenario}"
|
||||
assert "task" in scenario, f"Scenario missing 'task': {scenario}"
|
||||
assert "type" in scenario, (
|
||||
f"Scenario '{scenario['id']}' missing 'type' field. " f"Must be 'telepathic' or 'brain_action'."
|
||||
)
|
||||
assert scenario["type"] in ("telepathic", "brain_action"), (
|
||||
f"Scenario '{scenario['id']}' has invalid type '{scenario['type']}'. "
|
||||
f"Must be 'telepathic' or 'brain_action'."
|
||||
)
|
||||
|
||||
def test_brain_action_scenarios_exist(self):
|
||||
"""CRITICAL: Brain action extraction MUST be benchmarked."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
brain_scenarios = [s for s in data["scenarios"] if s.get("type") == "brain_action"]
|
||||
assert len(brain_scenarios) >= 3, (
|
||||
f"Only {len(brain_scenarios)} brain_action scenarios found. "
|
||||
f"Need at least 3 to reliably evaluate Brain action extraction."
|
||||
)
|
||||
|
||||
def test_brain_scenarios_have_available_actions(self):
|
||||
"""Brain scenarios must provide available_actions list."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
if scenario.get("type") != "brain_action":
|
||||
continue
|
||||
assert "available_actions" in scenario, f"Brain scenario '{scenario['id']}' missing 'available_actions'"
|
||||
assert "target_action" in scenario, f"Brain scenario '{scenario['id']}' missing 'target_action'"
|
||||
assert scenario["target_action"] in scenario["available_actions"], (
|
||||
f"Brain scenario '{scenario['id']}': target_action "
|
||||
f"'{scenario['target_action']}' not in available_actions"
|
||||
)
|
||||
|
||||
def test_telepathic_scenarios_have_nodes(self):
|
||||
"""Telepathic scenarios must provide nodes and target_index."""
|
||||
with open(SCENARIOS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for scenario in data["scenarios"]:
|
||||
if scenario.get("type") != "telepathic":
|
||||
continue
|
||||
assert "nodes" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'nodes'"
|
||||
assert "target_index" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'target_index'"
|
||||
|
||||
|
||||
class TestBenchmarkResultsIntegrity:
|
||||
"""Contract: Stored results must be consistent and comparable."""
|
||||
|
||||
@pytest.fixture
|
||||
def results(self):
|
||||
if not os.path.exists(RESULTS_FILE):
|
||||
pytest.skip("No benchmark results file yet")
|
||||
with open(RESULTS_FILE) as f:
|
||||
return json.load(f)
|
||||
|
||||
def test_details_format_is_consistent(self, results):
|
||||
"""All model details must use the same format (object, not raw int)."""
|
||||
for model_name, data in results.get("models", {}).items():
|
||||
details = data.get("details", {})
|
||||
for scenario_id, value in details.items():
|
||||
assert isinstance(value, dict), (
|
||||
f"Model '{model_name}' scenario '{scenario_id}' uses "
|
||||
f"legacy format (raw int: {value}). Must be "
|
||||
f"{{'avg_score': int, 'pass_rate': float, 'latency': int}}"
|
||||
)
|
||||
|
||||
def test_relative_performance_is_normalized(self, results):
|
||||
"""Relative performance must not exceed 100% (the leader)."""
|
||||
for model_name, data in results.get("models", {}).items():
|
||||
pct = data.get("relative_performance_pct", 0)
|
||||
assert pct <= 100.0, (
|
||||
f"Model '{model_name}' has relative_performance_pct={pct}% > 100%. "
|
||||
f"Scoring is not normalized by scenario count!"
|
||||
)
|
||||
30
tests/unit/test_bot_flow_autonomy.py
Normal file
30
tests/unit/test_bot_flow_autonomy.py
Normal file
@@ -0,0 +1,30 @@
|
||||
def test_bot_flow_uses_goal_decomposer_not_abstract_goals():
|
||||
"""
|
||||
Test that bot_flow.py uses GoalDecomposer for task-based navigation
|
||||
instead of the abstract goals string system.
|
||||
|
||||
The old system passed abstract strings like "Nurture my community"
|
||||
to GoalExecutor.achieve() — which caused endless scrolling because
|
||||
the LLM had no semantic bridge to concrete plugin actions.
|
||||
|
||||
The new system:
|
||||
1. GoalDecomposer reads mission + plugins → generates concrete Tasks
|
||||
2. GrowthBrain.select_task() picks a Task with weighted random
|
||||
3. The Task's target_screen routes through nav_graph to feed loops
|
||||
"""
|
||||
with open("GramAddict/core/bot_flow.py", "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# New system MUST be present
|
||||
assert "GoalDecomposer" in content, "bot_flow.py must use GoalDecomposer"
|
||||
assert "select_task" in content, "bot_flow.py must use GrowthBrain.select_task()"
|
||||
assert "available_tasks" in content, "bot_flow.py must generate available_tasks"
|
||||
|
||||
# Old abstract goals path MUST be gone
|
||||
assert "goal_executor.achieve(current_goal)" not in content, (
|
||||
"bot_flow.py still uses old abstract goal_executor.achieve()! "
|
||||
"This causes endless scrolling."
|
||||
)
|
||||
assert 'getattr(configs.args, "goals"' not in content, (
|
||||
"bot_flow.py still reads abstract goals from config!"
|
||||
)
|
||||
336
tests/unit/test_brain_output_contract.py
Normal file
336
tests/unit/test_brain_output_contract.py
Normal file
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
Brain Output Contract Tests — The Missing Guard
|
||||
================================================
|
||||
|
||||
These tests prove the CRITICAL pipeline:
|
||||
LLM raw output → Parser → Extracted action
|
||||
|
||||
This is the ROOT CAUSE of the 2026-04-28 production bug:
|
||||
The Brain's fuzzy matcher extracted 'tap messages tab' from the LLM's
|
||||
<think> block even though the LLM's conclusion was 'press back'.
|
||||
|
||||
TDD Rule: Every production bug gets a failing test FIRST.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
|
||||
class TestBrainOutputParsing:
|
||||
"""Contract: The Brain MUST extract the LLM's CONCLUSION, not mentioned words."""
|
||||
|
||||
def test_exact_match_wins(self, monkeypatch):
|
||||
"""When the LLM returns a clean, exact action string."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": "press back"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="open explore",
|
||||
screen_type="DM_INBOX",
|
||||
available_actions=["press back", "tap messages tab", "scroll down"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "press back"
|
||||
|
||||
def test_thinking_block_does_not_poison_extraction(self, monkeypatch):
|
||||
"""REGRESSION: The LLM mentions 'tap messages tab' in its reasoning
|
||||
but concludes with 'press back'. The parser MUST return 'press back'."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# This is the EXACT pattern from the production failure:
|
||||
verbose_thinking = (
|
||||
"The user wants to nurture their existing community. "
|
||||
"They're currently on the DM_INBOX screen. "
|
||||
"The previous action 'tap messages tab' failed, which is odd since "
|
||||
"we're already in DM_INBOX. Since I need to nurture the community, "
|
||||
"being in DM inbox is not the most effective place. "
|
||||
"The best action would be to exit the DM inbox. "
|
||||
"I should 'press back' to go to a different screen.\n\n"
|
||||
"press back"
|
||||
)
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": verbose_thinking}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="DM_INBOX",
|
||||
available_actions=["press back", "tap messages tab", "scroll down", "tap home tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "press back", (
|
||||
f"Brain extracted '{result}' instead of 'press back'. "
|
||||
f"The fuzzy matcher is poisoned by the <think> block!"
|
||||
)
|
||||
|
||||
def test_last_mentioned_action_wins_in_verbose_output(self, monkeypatch):
|
||||
"""When the LLM reasons through options, the LAST mentioned action is the decision."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
verbose_output = (
|
||||
"Let me think about this. I could 'scroll down' to see more content, "
|
||||
"or 'tap explore tab' to discover new posts. But since the goal is to "
|
||||
"find new accounts to engage with, I think 'tap explore tab' is the best choice."
|
||||
)
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": verbose_output}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="find accounts to engage",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "tap reels tab", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result == "tap explore tab", (
|
||||
f"Brain extracted '{result}' instead of 'tap explore tab'. "
|
||||
f"Expected the last-mentioned action to win."
|
||||
)
|
||||
|
||||
def test_brain_never_returns_avoided_action(self, monkeypatch):
|
||||
"""CRITICAL: Even if the LLM mentions an avoided action, the Brain must NOT return it."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
# LLM explicitly recommends the avoided action (Brain doesn't know about avoid_actions,
|
||||
# but the planner passes only non-masked actions as available_actions)
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": "tap messages tab"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
# 'tap messages tab' is NOT in available_actions (already masked by planner)
|
||||
result = ask_brain_for_action(
|
||||
goal="open messages",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "tap reels tab"],
|
||||
explored_actions={"tap messages tab"},
|
||||
)
|
||||
# The action MUST be None or one of the available actions — NEVER the masked one
|
||||
assert result != "tap messages tab", (
|
||||
"Brain returned an action that was not in available_actions! "
|
||||
"This means the masking layer has a hole."
|
||||
)
|
||||
|
||||
|
||||
class TestBrainAvoidActionsParity:
|
||||
"""Contract: The planner MUST strip avoided actions before passing to the Brain."""
|
||||
|
||||
def test_planner_masks_failed_actions_before_brain(self, monkeypatch):
|
||||
"""Verify the planner strips failed actions from the list BEFORE asking the Brain."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_available = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
# Capture the system prompt to verify available actions
|
||||
captured_available.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.DM_INBOX,
|
||||
"available_actions": ["tap messages tab", "press back", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step(
|
||||
"open explore",
|
||||
screen,
|
||||
action_failures={"tap messages tab": 2}, # Masked!
|
||||
)
|
||||
|
||||
assert len(captured_available) == 1, "Brain was not called"
|
||||
prompt = captured_available[0]
|
||||
|
||||
# Extract just the "available actions" line from the prompt
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
# The masked action must NOT be in the available actions list
|
||||
assert "tap messages tab" not in line, (
|
||||
f"Planner passed masked action 'tap messages tab' to the Brain as available!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in the Brain prompt")
|
||||
|
||||
|
||||
class TestUIChangedFidelity:
|
||||
"""Contract: Trivial XML diffs must NOT count as 'ui_changed'."""
|
||||
|
||||
def test_trivial_1_byte_diff_is_not_ui_change(self):
|
||||
"""REGRESSION: In the 2026-04-28 run, ui_changed=True with delta=1 byte
|
||||
(118399→118400). The GOAP then falsely confirmed the navigation as successful."""
|
||||
MIN_UI_CHANGE_BYTES = 50 # Must match the constant in goap.py
|
||||
|
||||
pre_xml = "x" * 118399
|
||||
post_xml = "x" * 118400
|
||||
xml_delta = abs(len(post_xml) - len(pre_xml))
|
||||
|
||||
# The production check
|
||||
ui_changed = pre_xml != post_xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is False, (
|
||||
f"1-byte diff (delta={xml_delta}) was treated as UI change! "
|
||||
f"This is the false-positive that caused the DM_INBOX loop."
|
||||
)
|
||||
|
||||
def test_large_diff_is_real_ui_change(self):
|
||||
"""A genuine screen transition changes the XML by hundreds/thousands of bytes."""
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
|
||||
pre_xml = "<hierarchy><node text='Home Feed' /></hierarchy>"
|
||||
post_xml = "<hierarchy><node text='Explore Grid' />" + "<node />" * 100 + "</hierarchy>"
|
||||
xml_delta = abs(len(post_xml) - len(pre_xml))
|
||||
|
||||
ui_changed = pre_xml != post_xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is True, f"Real UI change (delta={xml_delta}) was NOT detected!"
|
||||
|
||||
def test_identical_xml_is_not_ui_change(self):
|
||||
"""Exact same XML → no change."""
|
||||
xml = "<hierarchy><node text='Hello' /></hierarchy>"
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
xml_delta = abs(len(xml) - len(xml))
|
||||
ui_changed = xml != xml and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
assert ui_changed is False
|
||||
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType # noqa: E402
|
||||
|
||||
|
||||
class TestBrainEmptyResponse:
|
||||
"""Contract: When the LLM returns response='', the Brain must NOT
|
||||
extract actions from the thinking block. The thinking block is
|
||||
REASONING, not decisions."""
|
||||
|
||||
def test_empty_response_returns_none_not_thinking_extraction(self, monkeypatch):
|
||||
"""REGRESSION: In the 2026-04-28 23:51 run, the LLM returned response=''
|
||||
with a thinking block mentioning 'tap profile tab'. The Brain extracted
|
||||
'tap profile tab' which was a no-op on OWN_PROFILE."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
# The EXACT production failure: response is empty, thinking has actions
|
||||
return {"response": ""}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
# When the LLM gives NO response, the Brain must return None
|
||||
# to force the planner's structural fallback
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from an empty LLM response! "
|
||||
f"It must return None so the planner can use HD Map fallback."
|
||||
)
|
||||
|
||||
def test_whitespace_only_response_treated_as_empty(self, monkeypatch):
|
||||
"""Response with only whitespace/newlines is effectively empty."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": " \n \n "}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="open explore",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["tap explore tab", "scroll down"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from a whitespace-only response! "
|
||||
f"Must return None."
|
||||
)
|
||||
|
||||
|
||||
class TestPlannerNoOpGuard:
|
||||
"""Contract: The planner must NEVER ask the Brain to execute a tab action
|
||||
that would navigate to the screen we're already on."""
|
||||
|
||||
def test_planner_strips_current_screen_tab_before_brain(self, monkeypatch):
|
||||
"""On OWN_PROFILE, 'tap profile tab' is a no-op. The planner must
|
||||
strip it from available_actions before asking the Brain."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.OWN_PROFILE,
|
||||
"available_actions": ["tap profile tab", "tap home tab", "scroll down", "press back"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap profile tab" not in line, (
|
||||
f"Planner passed no-op action 'tap profile tab' to Brain on OWN_PROFILE!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
|
||||
def test_planner_strips_home_tab_on_home_feed(self, monkeypatch):
|
||||
"""On HOME_FEED, 'tap home tab' is a no-op."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["tap home tab", "tap explore tab", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap home tab" not in line, (
|
||||
f"Planner passed no-op 'tap home tab' to Brain on HOME_FEED!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
58
tests/unit/test_device_connection.py
Normal file
58
tests/unit/test_device_connection.py
Normal file
@@ -0,0 +1,58 @@
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.device_facade import create_device
|
||||
|
||||
|
||||
def test_create_device_connection_failure(monkeypatch, caplog):
|
||||
"""Test that create_device handles connection failures gracefully by logging and exiting."""
|
||||
|
||||
def mock_connect_fail(device_id):
|
||||
# Simulate a uiautomator2 connection failure
|
||||
raise Exception(
|
||||
"ConnectError: [WinError 10061] No connection could be made because the target machine actively refused it"
|
||||
)
|
||||
|
||||
import subprocess
|
||||
from collections import namedtuple
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import uiautomator2 as u2
|
||||
|
||||
monkeypatch.setattr(u2, "connect", mock_connect_fail)
|
||||
|
||||
# Mock subprocess.run for "adb devices"
|
||||
CompletedProcess = namedtuple("CompletedProcess", ["stdout", "stderr", "returncode"])
|
||||
# Case 2: Proactive discovery with NO devices
|
||||
monkeypatch.setattr(
|
||||
subprocess, "run", lambda *args, **kwargs: MagicMock(stdout="List of devices attached\n\n", return_code=0)
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
create_device("192.168.1.100:5555", "com.instagram.android", None)
|
||||
|
||||
# Case 3: Proactive discovery with MISMATCHED IP
|
||||
monkeypatch.setattr(
|
||||
subprocess,
|
||||
"run",
|
||||
lambda *args, **kwargs: MagicMock(stdout="List of devices attached\n10.0.0.5:5555\tdevice\n", return_code=0),
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
create_device("192.168.1.100:5555", "com.instagram.android", None)
|
||||
|
||||
def mock_adb_devices(*args, **kwargs):
|
||||
# Simulate output where the IP matches but the port is different
|
||||
return CompletedProcess(
|
||||
stdout="List of devices attached\n192.168.1.206:34771\tdevice\n", stderr="", returncode=0
|
||||
)
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", mock_adb_devices)
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
create_device("192.168.1.206:35911", "com.instagram.android")
|
||||
|
||||
assert excinfo.value.code == 1
|
||||
assert "[ADB ConnectError]" in caplog.text
|
||||
assert "🔍 Proactive Discovery" in caplog.text
|
||||
assert "192.168.1.206:34771 (MATCHING IP - Is this the same device with a different port?)" in caplog.text
|
||||
@@ -1,101 +0,0 @@
|
||||
"""
|
||||
🔴 RED TDD: DM Structural Guard Self-Sabotage Fix
|
||||
|
||||
Reproduces Bug 2: The intent 'tap direct message icon inbox' is NOT classified
|
||||
as a nav intent, causing the Structural Guard to reject the correct VLM match
|
||||
in the nav bar zone.
|
||||
|
||||
These tests MUST FAIL before the fix and PASS after.
|
||||
"""
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
class TestNavIntentClassification:
|
||||
"""Verifies that all navigation-related intents are correctly classified."""
|
||||
|
||||
def test_dm_intent_is_classified_as_nav_intent(self):
|
||||
"""
|
||||
The intent 'tap direct message icon inbox' MUST be treated as a nav intent
|
||||
so the structural guard allows clicking elements in the nav bar zone.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# DM icon is in the nav bar zone (top right, but the 'direct tab'
|
||||
# element is at the bottom nav bar on some Instagram layouts)
|
||||
dm_node = {
|
||||
"semantic_string": "description: 'Message', id context: 'direct tab'",
|
||||
"y": int(screen_height * 0.95), # Bottom nav bar zone
|
||||
"area": 3000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "direct_tab",
|
||||
}
|
||||
|
||||
intent = "tap direct message icon inbox"
|
||||
|
||||
# The node should be viable — it's a nav intent targeting the nav bar
|
||||
is_valid = engine._structural_sanity_check(dm_node, intent, screen_height)
|
||||
|
||||
assert is_valid is True, (
|
||||
"Structural Guard rejected 'direct tab' for DM intent. "
|
||||
"This is the exact bug: 'tap direct message icon inbox' is not classified as nav intent."
|
||||
)
|
||||
|
||||
def test_inbox_intent_is_classified_as_nav_intent(self):
|
||||
"""Variant: 'tap inbox' should also be treated as navigation."""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
inbox_node = {
|
||||
"semantic_string": "description: 'Inbox', id context: 'direct_inbox'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2500,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "direct_inbox",
|
||||
}
|
||||
|
||||
intent = "tap inbox"
|
||||
|
||||
is_valid = engine._structural_sanity_check(inbox_node, intent, screen_height)
|
||||
assert is_valid is True, "Structural Guard rejected inbox node for 'tap inbox' intent."
|
||||
|
||||
def test_notification_intent_is_classified_as_nav_intent(self):
|
||||
"""'tap heart icon notifications' should also be treated as navigation."""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
notification_node = {
|
||||
"semantic_string": "description: 'Activity', id context: 'notification_tab'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2500,
|
||||
"class_name": "android.widget.ImageView",
|
||||
"resource_id": "notification_tab",
|
||||
}
|
||||
|
||||
intent = "tap heart icon notifications"
|
||||
|
||||
is_valid = engine._structural_sanity_check(notification_node, intent, screen_height)
|
||||
assert is_valid is True, "Structural Guard rejected notification node for heart icon intent."
|
||||
|
||||
def test_regular_post_intent_still_blocked_in_nav_zone(self):
|
||||
"""
|
||||
Non-nav intents (like 'tap like button') targeting elements in the nav bar
|
||||
zone must STILL be rejected. We're not weakening the guard.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
misplaced_like_node = {
|
||||
"semantic_string": "description: 'Like', id context: 'some_like_button'",
|
||||
"y": int(screen_height * 0.95),
|
||||
"area": 2000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
intent = "tap like button"
|
||||
|
||||
is_valid = engine._structural_sanity_check(misplaced_like_node, intent, screen_height)
|
||||
assert is_valid is False, (
|
||||
"Structural Guard allowed a like button in the nav bar zone. " "Non-nav intents should still be blocked."
|
||||
)
|
||||
@@ -1,150 +0,0 @@
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
def test_structural_guard_rejects_own_story_for_post_username():
|
||||
"""
|
||||
TDD Test: Reproduces the bug where Telepathic Engine might select the user's
|
||||
OWN profile picture ("Your Story" in the Home Feed tray) when the intent
|
||||
is to tap the post author's username.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Mock node representing the user's "Your Story" circle at the top
|
||||
# It contains "story" or "your story", has low Y (top of screen)
|
||||
your_story_node = {
|
||||
"semantic_string": "description: 'Your Story', id context: 'row feed photo profile imageview'",
|
||||
"y": 250, # Top story tray
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
# Intent
|
||||
intent = "tap post username"
|
||||
|
||||
# Expected behavior: Structural sanity check must REJECT this node to prevent
|
||||
# clicking our own story/profile
|
||||
is_valid = engine._structural_sanity_check(your_story_node, intent, screen_height)
|
||||
|
||||
assert is_valid is False, "Structural Guard failed to reject 'Your Story' when looking for 'post username'."
|
||||
|
||||
|
||||
def test_structural_guard_accepts_actual_post_username():
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
actual_post_node = {
|
||||
"semantic_string": "text: 'estherabad9', id context: 'row feed photo profile name'",
|
||||
"y": 1200, # Middle of screen (feed post header)
|
||||
"area": 5000,
|
||||
"class_name": "android.widget.TextView",
|
||||
}
|
||||
|
||||
intent = "tap post username"
|
||||
|
||||
is_valid = engine._structural_sanity_check(actual_post_node, intent, screen_height)
|
||||
|
||||
assert is_valid is True, "Structural Guard incorrectly rejected the actual post username."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_own_username_story():
|
||||
"""
|
||||
TDD Test: Reproduces 2026-04-16 23:18 bug where bot selected 'marisaundmarc's story'
|
||||
instead of an unseen story from ANOTHER user.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Simulate current user is marisaundmarc
|
||||
engine._get_current_username = lambda: "marisaundmarc"
|
||||
|
||||
# Mock node representing the user's OWN story, which contains their username
|
||||
own_story_node = {
|
||||
"semantic_string": "description: 'marisaundmarc\\'s story, 0 of 27, Unseen.', id context: 'avatar image view'",
|
||||
"y": 250, # Top story tray
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
intent = "profile picture avatar story ring"
|
||||
|
||||
# Should reject the user's own profile because clicking it means we edit/view our own story
|
||||
# instead of doing interactions with prospects.
|
||||
is_valid = engine._structural_sanity_check(own_story_node, intent, screen_height)
|
||||
|
||||
assert is_valid is False, "Structural Guard failed to reject the bot's OWN username story."
|
||||
|
||||
|
||||
def test_structural_reels_first_grid_item_y_coords():
|
||||
"""
|
||||
TDD Test: Reels viewer layout has grid items that are structurally valid.
|
||||
Ensures that relative Y coordinates (percentage of screen height) correctly
|
||||
allow valid grid items and block hallucinations.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2400
|
||||
|
||||
# Valid first grid item in a profile's reel tab, usually around y=700 to 1200
|
||||
valid_grid_node = {
|
||||
"semantic_string": "description: 'reel, 1 of 20', id context: 'image button'",
|
||||
"y": 800, # well within safe zone, ~33%
|
||||
"area": 40000,
|
||||
"class_name": "android.widget.ImageView",
|
||||
}
|
||||
|
||||
# Hallucinated navigation tab node pretending to be "Home" around y=1200 (middle of screen)
|
||||
hallucinated_nav_node = {
|
||||
"semantic_string": "description: 'Home', id context: 'tab'",
|
||||
"y": 1200, # 50% height
|
||||
"area": 1000,
|
||||
"class_name": "android.view.View",
|
||||
}
|
||||
|
||||
intent_grid = "first grid item"
|
||||
intent_nav = "tap home tab"
|
||||
|
||||
is_valid_grid = engine._structural_sanity_check(valid_grid_node, intent_grid, screen_height)
|
||||
assert is_valid_grid is True, "Structural Guard rejected a valid reels grid item."
|
||||
|
||||
# The hallucinated nav node should be rejected because navigation tabs belong at the bottom!
|
||||
# Currently it might fail if we don't have relative coordinate checks!
|
||||
is_valid_nav = engine._structural_sanity_check(hallucinated_nav_node, intent_nav, screen_height)
|
||||
assert (
|
||||
is_valid_nav is False
|
||||
), "Structural Guard failed to reject a hallucinated navigation tab in the middle of the screen."
|
||||
|
||||
def test_structural_guard_rejects_search_keyword_for_media_content():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "text: 'i\\'m', id context: 'row search keyword title'",
|
||||
"class_name": "android.widget.TextView",
|
||||
"y": 500
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "post media content", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject 'row_search_keyword_title' for 'post media content'."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_search_user_for_post_username():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "desc: 'Followed by pratiek_the_entrepreneur + 19 more', id context: 'row search user container'",
|
||||
"class_name": "android.widget.LinearLayout",
|
||||
"y": 800
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "tap post username", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject 'row_search_user_container' for 'tap post username'."
|
||||
|
||||
|
||||
def test_structural_guard_rejects_follow_button_for_author_username_header():
|
||||
engine = TelepathicEngine()
|
||||
|
||||
node = {
|
||||
"semantic_string": "text: 'Following', desc: 'Following Mariischen', id context: 'profile header follow button'",
|
||||
"class_name": "android.widget.Button",
|
||||
"y": 600
|
||||
}
|
||||
|
||||
is_valid = engine._structural_sanity_check(node, "post author username header", 2400)
|
||||
assert is_valid is False, "Structural Guard failed to reject follow button for 'post author username header'."
|
||||
@@ -14,7 +14,7 @@ class TestVerifySuccessGridReels:
|
||||
self.engine = TelepathicEngine()
|
||||
# Simulate a click context so verify_success has something to check against
|
||||
TelepathicEngine._last_click_context = {
|
||||
"intent": "first image in explore grid",
|
||||
"intent": "view a post",
|
||||
"semantic_string": "id context: 'image button'",
|
||||
"x": 178,
|
||||
"y": 558,
|
||||
@@ -32,7 +32,7 @@ class TestVerifySuccessGridReels:
|
||||
<node content-desc="Comment" resource-id="com.instagram.android:id/clips_comment_button" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", reel_xml)
|
||||
result = self.engine.verify_success("view a post", reel_xml)
|
||||
assert result is True, "verify_success rejected a valid Reel view opened from grid tap"
|
||||
|
||||
def test_normal_feed_post_still_accepted(self):
|
||||
@@ -44,7 +44,7 @@ class TestVerifySuccessGridReels:
|
||||
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="@testuser" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", feed_xml)
|
||||
result = self.engine.verify_success("view a post", feed_xml)
|
||||
assert result is True, "verify_success rejected a valid feed post opened from grid tap"
|
||||
|
||||
def test_explore_grid_still_visible_is_failure(self):
|
||||
@@ -57,17 +57,17 @@ class TestVerifySuccessGridReels:
|
||||
<node text="Search" resource-id="com.instagram.android:id/action_bar_search_edit_text" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image in explore grid", explore_xml)
|
||||
result = self.engine.verify_success("view a post", explore_xml)
|
||||
assert result is None, "verify_success should return None (inconclusive) when grid is still visible"
|
||||
|
||||
def test_profile_grid_reel_accepted(self):
|
||||
"""Profile grid → Reel must also be accepted."""
|
||||
TelepathicEngine._last_click_context["intent"] = "first image post in profile grid"
|
||||
TelepathicEngine._last_click_context["intent"] = "view a post"
|
||||
reel_xml = """
|
||||
<hierarchy>
|
||||
<node resource-id="com.instagram.android:id/clips_viewer_view_pager" />
|
||||
<node resource-id="com.instagram.android:id/reel_viewer_subtitle" text="Audio" />
|
||||
</hierarchy>
|
||||
"""
|
||||
result = self.engine.verify_success("first image post in profile grid", reel_xml)
|
||||
result = self.engine.verify_success("view a post", reel_xml)
|
||||
assert result is True, "verify_success rejected a Reel opened from profile grid"
|
||||
|
||||
Reference in New Issue
Block a user