feat(navigation): harden obstacle guard and intent resolver against keyboard hallucinations
This commit is contained in:
@@ -157,7 +157,7 @@ class PluginRegistry:
|
||||
|
||||
self._plugins.append(plugin)
|
||||
self._sorted = False
|
||||
logger.info(f"🧩 [Plugin] Registered: {plugin.name} (priority={plugin.priority})")
|
||||
logger.debug(f"🧩 [Plugin] Registered: {plugin.name} (priority={plugin.priority})")
|
||||
|
||||
def unregister(self, name: str):
|
||||
"""Remove a plugin by name."""
|
||||
@@ -206,6 +206,10 @@ class PluginRegistry:
|
||||
logger.debug(f"🧩 [PluginRegistry] TRACE: Calling execute() on {plugin.name}")
|
||||
result = plugin.execute(ctx)
|
||||
results.append(result)
|
||||
if result.executed:
|
||||
logger.debug(
|
||||
f"🧩 [PluginRegistry] Plugin {plugin.name} executed successfully. Metadata: {result.metadata}"
|
||||
)
|
||||
|
||||
if (plugin.exclusive and result.executed) or result.should_skip:
|
||||
logger.debug(
|
||||
|
||||
@@ -32,18 +32,16 @@ class CommentPlugin(BehaviorPlugin):
|
||||
if ctx.session_state.check_limit(SessionState.Limit.COMMENTS):
|
||||
return False
|
||||
|
||||
# Safety Guard: Do not comment on stories or grids
|
||||
xml_lower = (ctx.context_xml or "").lower()
|
||||
STORY_MARKERS = (
|
||||
"reel_viewer_media_layout",
|
||||
"reel_viewer_header",
|
||||
"reel_viewer_progress_bar",
|
||||
"reel_viewer_root",
|
||||
)
|
||||
if any(marker in xml_lower for marker in STORY_MARKERS):
|
||||
return False
|
||||
# ── STRUCTURAL GUARD ──
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
if "explore_action_bar" in xml_lower or "profile_tabs_container" in xml_lower:
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
if screen_type not in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED):
|
||||
return False
|
||||
|
||||
config = self.get_config(ctx)
|
||||
|
||||
@@ -30,6 +30,17 @@ class DarwinDwellPlugin(BehaviorPlugin):
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
|
||||
if screen_type not in valid_screens:
|
||||
return False
|
||||
|
||||
config = self.get_config(ctx)
|
||||
percentage = float(config.get("percentage", 100))
|
||||
return random.random() < (percentage / 100.0)
|
||||
|
||||
@@ -35,6 +35,18 @@ class FollowPlugin(BehaviorPlugin):
|
||||
if ctx.session_state.check_limit(SessionState.Limit.FOLLOWS):
|
||||
return False
|
||||
|
||||
# ── STRUCTURAL GUARD ──
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
if screen_type not in (ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST):
|
||||
return False
|
||||
|
||||
# Probability gate
|
||||
if random.random() >= follow_pct:
|
||||
return False
|
||||
@@ -49,8 +61,8 @@ class FollowPlugin(BehaviorPlugin):
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if nav_graph.do("tap 'Follow' button") or nav_graph.do("tap 'Following' button"):
|
||||
logger.info(f"🤝 [Follow] Followed @{ctx.username} ✓")
|
||||
if nav_graph.do("tap 'Follow' or 'Following' button"):
|
||||
logger.info(f"🤝 [Follow] Toggled Follow/Following state for @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
|
||||
|
||||
# Buffer for follow animations to close
|
||||
|
||||
@@ -38,14 +38,15 @@ class GridLikePlugin(BehaviorPlugin):
|
||||
return False
|
||||
|
||||
# ── STRUCTURAL GUARD ──
|
||||
nav_graph = ctx.cognitive_stack.get("nav_graph")
|
||||
if nav_graph and nav_graph.current_state != "ProfileView":
|
||||
return False
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
# Fallback XML check
|
||||
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
|
||||
xml_lower = xml.lower()
|
||||
if "followers" not in xml_lower and "beiträge" not in xml_lower and "posts" not in xml_lower:
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
if screen_type not in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE, ScreenType.EXPLORE_GRID):
|
||||
return False
|
||||
|
||||
# Probability gate
|
||||
@@ -75,7 +76,20 @@ class GridLikePlugin(BehaviorPlugin):
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if not nav_graph.do("tap first image post in profile grid"):
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
nav_action = (
|
||||
"tap first image in explore grid"
|
||||
if screen_type == ScreenType.EXPLORE_GRID
|
||||
else "tap first image post in profile grid"
|
||||
)
|
||||
|
||||
if not nav_graph.do(nav_action):
|
||||
return BehaviorResult(executed=False, metadata={"reason": "grid_nav_failed"})
|
||||
|
||||
if not wait_for_post_loaded(ctx.device, timeout=5, nav_graph=nav_graph):
|
||||
|
||||
@@ -39,6 +39,18 @@ class LikePlugin(BehaviorPlugin):
|
||||
if likes_pct <= 0:
|
||||
return False
|
||||
|
||||
# ── STRUCTURAL GUARD ──
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
if screen_type not in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED):
|
||||
return False
|
||||
|
||||
# Probability gate
|
||||
if random.random() >= likes_pct:
|
||||
return False
|
||||
|
||||
@@ -56,6 +56,13 @@ class ObstacleGuardPlugin(BehaviorPlugin):
|
||||
sleep(1.5 * ctx.sleep_mod)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
|
||||
# ── On-Screen Keyboard (e.g. hallucinated click on comment field) ──
|
||||
if situation == SituationType.OBSTACLE_KEYBOARD:
|
||||
logger.warning("⚠️ [ObstacleGuard] On-screen Keyboard is open. Pressing BACK to dismiss...")
|
||||
ctx.device.press("back")
|
||||
sleep(1.0 * ctx.sleep_mod)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
|
||||
# ── Instagram Modal / Overlay (survey, "Not Now" prompt, creation flow) ──
|
||||
if situation == SituationType.OBSTACLE_MODAL:
|
||||
if misses >= 2:
|
||||
|
||||
@@ -31,7 +31,18 @@ class PostInteractionPlugin(BehaviorPlugin):
|
||||
return True # Ends the behavior chain for this post
|
||||
|
||||
def can_activate(self, ctx: BehaviorContext) -> bool:
|
||||
return getattr(self, "_enabled", True)
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
|
||||
return screen_type in valid_screens
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
logger.info("🏁 [PostInteraction] Interactions complete. Moving to next post...")
|
||||
|
||||
@@ -31,12 +31,24 @@ class ProfileVisitPlugin(BehaviorPlugin):
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
# 1. Guard against recursive calls or being already on profile
|
||||
# 1. Screen Guard: Only activate on feed screens
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
valid_screens = [ScreenType.HOME_FEED, ScreenType.EXPLORE_GRID, ScreenType.REELS_FEED]
|
||||
if screen_type not in valid_screens:
|
||||
return False
|
||||
|
||||
# 2. Guard against recursive calls or being already on profile
|
||||
nav_graph = ctx.cognitive_stack.get("nav_graph")
|
||||
if nav_graph and nav_graph.current_state == "ProfileView":
|
||||
return False
|
||||
|
||||
# 2. Probability gate
|
||||
# 3. Probability gate
|
||||
config = self.get_config(ctx)
|
||||
visit_pct = float(config.get("percentage", getattr(ctx.configs.args, "profile_visit_percentage", 30))) / 100.0
|
||||
|
||||
|
||||
@@ -30,6 +30,18 @@ class RepostPlugin(BehaviorPlugin):
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
# 1. Screen Guard: Only activate on post-containing screens
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED]
|
||||
if screen_type not in valid_screens:
|
||||
return False
|
||||
|
||||
config = self.get_config(ctx)
|
||||
repost_pct = float(config.get("percentage", getattr(ctx.configs.args, "repost_percentage", 20))) / 100.0
|
||||
|
||||
|
||||
@@ -27,7 +27,17 @@ class ResonanceEvaluatorPlugin(BehaviorPlugin):
|
||||
return 80
|
||||
|
||||
def can_activate(self, ctx: BehaviorContext) -> bool:
|
||||
return getattr(self, "_enabled", True)
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
screen_type = ctx.shared_state.get("current_screen_type")
|
||||
if not screen_type and ctx.context_xml:
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
|
||||
return screen_type in valid_screens
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
resonance = ctx.cognitive_stack.get("resonance")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from time import sleep
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext, BehaviorPlugin, BehaviorResult
|
||||
@@ -96,8 +97,39 @@ class StoryViewPlugin(BehaviorPlugin):
|
||||
sleep(random.uniform(2.0, 5.0) * ctx.sleep_mod)
|
||||
if i < count - 1:
|
||||
humanized_click(ctx.device, int(w * 0.9), int(h * 0.5), sleep_mod=ctx.sleep_mod)
|
||||
# Atomic state validation after click
|
||||
xml_dump = ctx.device.dump_hierarchy()
|
||||
if not xml_dump:
|
||||
continue
|
||||
packages = set(re.findall(r'package="([^"]+)"', xml_dump))
|
||||
app_id = getattr(ctx.device, "app_id", "com.instagram.android")
|
||||
if packages and app_id not in packages:
|
||||
logger.error(
|
||||
f"🚨 [StoryView] FOREIGN APP DETECTED! Packages: {packages}. "
|
||||
f"A link likely opened an external app. Aborting loop."
|
||||
)
|
||||
ctx.device.press("back")
|
||||
sleep(1.5)
|
||||
break
|
||||
|
||||
ctx.device.press("back")
|
||||
sleep(random.uniform(1.0, 2.0) * ctx.sleep_mod)
|
||||
|
||||
# Post-interaction verification: verify we successfully exited the story overlay
|
||||
for attempt in range(3):
|
||||
xml_dump = ctx.device.dump_hierarchy()
|
||||
if not xml_dump:
|
||||
break
|
||||
xml_lower = xml_dump.lower()
|
||||
if "com.instagram.android" not in xml_dump or (
|
||||
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
|
||||
):
|
||||
# Successfully back to a main view or outside instagram
|
||||
break
|
||||
logger.warning(
|
||||
f"⚠️ [StoryView] Still trapped in story/overlay after back press (attempt {attempt+1}). Pressing back again."
|
||||
)
|
||||
ctx.device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
return BehaviorResult(executed=True, interactions=count, metadata={"stories_viewed": count})
|
||||
|
||||
@@ -621,6 +621,24 @@ def start_bot(**kwargs):
|
||||
break
|
||||
|
||||
logger.info(f"Session complete. Boredom: {dopamine.boredom:.1f}%. Sleeping before next iteration...")
|
||||
|
||||
# ── P1-3: Wire EvolutionEngine ──
|
||||
try:
|
||||
from GramAddict.core.evolution_engine import EvolutionEngine, SessionResult
|
||||
|
||||
evolution_engine = EvolutionEngine.get_instance(username)
|
||||
session_result = SessionResult(
|
||||
follows_gained=sum(session_state.totalFollowed.values()),
|
||||
likes_given=session_state.totalLikes,
|
||||
stories_viewed=getattr(session_state, "totalWatched", 0),
|
||||
blocks_received=getattr(session_state, "totalBlocks", 0),
|
||||
duration_minutes=(datetime.now() - session_state.startTime).total_seconds() / 60.0,
|
||||
profiles_scraped=getattr(session_state, "totalScraped", 0),
|
||||
)
|
||||
evolution_engine.evolve(session_result)
|
||||
except Exception as e:
|
||||
logger.error(f"⚠️ Failed to run EvolutionEngine: {e}")
|
||||
|
||||
close_instagram(device)
|
||||
random_sleep(30, 60)
|
||||
|
||||
@@ -872,7 +890,23 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
# Recover gracefully instead of forcing a nuclear app restart
|
||||
for attempt in range(3):
|
||||
xml_dump = device.dump_hierarchy()
|
||||
if not xml_dump:
|
||||
break
|
||||
xml_lower = xml_dump.lower()
|
||||
if app_id in xml_dump and (
|
||||
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
|
||||
):
|
||||
break
|
||||
logger.warning(
|
||||
f"⚠️ [StoriesFeed] Still trapped after back press (attempt {attempt+1}). Pressing back again."
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
return "FEED_EXHAUSTED"
|
||||
|
||||
if getattr(configs.args, "ignore_close_friends", False):
|
||||
if "enge freunde" in xml_dump.lower() or "close friend" in xml_dump.lower():
|
||||
@@ -892,6 +926,24 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
|
||||
|
||||
logger.info("🎬 [StoriesFeed] Session completed naturally.")
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
# ── Strict Navigation-Verification ──
|
||||
for attempt in range(3):
|
||||
xml_dump = device.dump_hierarchy()
|
||||
if not xml_dump:
|
||||
break
|
||||
xml_lower = xml_dump.lower()
|
||||
if "com.instagram.android" in xml_dump and (
|
||||
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
|
||||
):
|
||||
break
|
||||
logger.warning(
|
||||
f"⚠️ [StoriesFeed] Still trapped in story/overlay after back press (attempt {attempt+1}). Pressing back again."
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
return "FEED_EXHAUSTED"
|
||||
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ class DopamineEngine:
|
||||
return True
|
||||
|
||||
# True if we have scrolled too long or hit absolute burnout
|
||||
return (time.time() - self.session_start) > self.session_limit_seconds or self.boredom >= 100.0
|
||||
return (time.time() - self.session_start) >= self.session_limit_seconds or self.boredom >= 100.0
|
||||
|
||||
def get_pacing_modifier(self, base_score: float):
|
||||
"""
|
||||
|
||||
@@ -375,25 +375,6 @@ class GoalExecutor:
|
||||
self._get_sae().ensure_clear_screen(max_attempts=3)
|
||||
return False
|
||||
|
||||
# ── Pre-Click Semantic Match Guard ──
|
||||
# For toggle intents (follow/like/save), verify the selected node
|
||||
# semantically matches the intent BEFORE clicking. This prevents
|
||||
# VLM hallucinations from clicking photo grid items when looking
|
||||
# for follow buttons.
|
||||
from GramAddict.core.perception.action_memory import _intent_matches_node
|
||||
|
||||
node_semantic = (
|
||||
f"text: '{best_node.get('text', '')}', "
|
||||
f"desc: '{best_node.get('description', '')}', "
|
||||
f"id: '{best_node.get('id', '')}'"
|
||||
)
|
||||
if not _intent_matches_node(action, node_semantic):
|
||||
logger.warning(
|
||||
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
|
||||
f"Node: {node_semantic}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Execute click
|
||||
self.device.click(obj=best_node)
|
||||
import random
|
||||
|
||||
@@ -53,16 +53,16 @@ def ask_brain_for_action(
|
||||
)
|
||||
if response:
|
||||
result = response if isinstance(response, str) else response.get("response", "")
|
||||
result = result.strip().strip("'\"")
|
||||
result = result.strip().strip("'\"").rstrip(".")
|
||||
|
||||
# 1. Exact match check (ideal case)
|
||||
for act in available_actions:
|
||||
if act.lower() == result.lower():
|
||||
return act
|
||||
|
||||
|
||||
# 2. Strict line-by-line check (often the model outputs the action on the last line)
|
||||
for line in reversed(result.splitlines()):
|
||||
line = line.strip().strip("'\"")
|
||||
line = line.strip().strip("'\"").rstrip(".")
|
||||
for act in available_actions:
|
||||
if act.lower() == line.lower():
|
||||
return act
|
||||
@@ -75,12 +75,14 @@ def ask_brain_for_action(
|
||||
if idx > best_idx:
|
||||
best_idx = idx
|
||||
best_act = act
|
||||
|
||||
|
||||
if best_act:
|
||||
logger.warning(f"🧠 [Brain] Extracted action '{best_act}' from verbose LLM output.")
|
||||
return best_act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back.")
|
||||
logger.warning(
|
||||
f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"🧠 [Brain] Error querying LLM: {e}")
|
||||
|
||||
|
||||
@@ -186,6 +186,11 @@ class NavigationKnowledge:
|
||||
|
||||
def is_trap(self, screen_type: ScreenType, action: str) -> bool:
|
||||
"""Check if an action on this screen is a known trap."""
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
if ScreenTopology.is_structural_action(screen_type, action):
|
||||
return False # Structural actions can NEVER be traps
|
||||
|
||||
trap_key = f"{screen_type.name}_{action}"
|
||||
if trap_key in self._learned_traps:
|
||||
return True
|
||||
|
||||
@@ -178,6 +178,36 @@ class GoalPlanner:
|
||||
f"🛡️ [HD Map] Route action '{next_action}' already explored and failed. Skipping HD Map."
|
||||
)
|
||||
|
||||
# ── 2.5. ContextGate Feedback Loop ──
|
||||
# Preempt the brain from hallucinating banned interaction intents.
|
||||
from GramAddict.core.perception.context_gate import ContextGate
|
||||
|
||||
cg = ContextGate()
|
||||
valid_screens = cg.get_valid_screens(goal)
|
||||
if valid_screens is not None and screen_type not in valid_screens:
|
||||
logger.warning(
|
||||
f"🛡️ [Planner Feedback] Goal '{goal}' is structurally banned on {screen_type.name} by ContextGate."
|
||||
)
|
||||
# We are trapped from doing the goal here. Must navigate to one of the valid screens.
|
||||
best_route = None
|
||||
for vs in valid_screens:
|
||||
r = ScreenTopology.find_route(screen_type, vs, avoid_actions=avoid_actions)
|
||||
if r and (not best_route or len(r) < len(best_route)):
|
||||
best_route = r
|
||||
if best_route:
|
||||
next_action, next_screen = best_route[0]
|
||||
if next_action not in (explored_nav_actions or set()):
|
||||
if not self.knowledge.is_trap(screen_type, next_action):
|
||||
logger.info(
|
||||
f"🗺️ [Planner Feedback] Auto-routing to {best_route[-1][1].name} via '{next_action}'"
|
||||
)
|
||||
return next_action
|
||||
|
||||
# If no route found, force back-tracking or skip brain to avoid hallucination.
|
||||
if "press back" in available:
|
||||
return "press back"
|
||||
return None
|
||||
|
||||
# ── 3. Brain-Driven Decision Making (Fallback / Discovery) ──
|
||||
# For non-navigation goals or when the HD Map is incomplete.
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
@@ -39,21 +39,8 @@ def _parse_yes_no(response: str) -> Optional[bool]:
|
||||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Semantic Match Keywords — SSOT for intent → element validation
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
# Maps toggle-intent keywords to required element markers.
|
||||
# If the intent contains the key, the clicked element MUST
|
||||
# contain at least one of the corresponding markers in its
|
||||
# text, content_desc, or resource_id.
|
||||
# ZERO MAINTENANCE: Only English words and resource_id fragments allowed.
|
||||
# No localized strings — the bot must work on any device language.
|
||||
TOGGLE_INTENT_MARKERS = {
|
||||
"follow": ["follow", "button_follow"],
|
||||
"like": ["like", "heart", "button_like"],
|
||||
"save": ["save", "saved", "bookmark"],
|
||||
}
|
||||
# FSD Architecture: No static string dictionaries.
|
||||
# The bot relies 100% on learned confidence and VLM/Delta verification.
|
||||
|
||||
|
||||
class ActionMemory:
|
||||
@@ -62,8 +49,8 @@ class ActionMemory:
|
||||
Decouples the memory layer from the core parsing engine.
|
||||
"""
|
||||
|
||||
def __init__(self, ui_memory=None):
|
||||
# We optionally inject UIMemoryDB to decouple tests
|
||||
def __init__(self, ui_memory=None, context_memory=None):
|
||||
# We optionally inject UIMemoryDB and ContextMemoryDB to decouple tests
|
||||
if ui_memory is None:
|
||||
from GramAddict.core.qdrant_memory import UIMemoryDB
|
||||
|
||||
@@ -71,9 +58,16 @@ class ActionMemory:
|
||||
else:
|
||||
self.ui_memory = ui_memory
|
||||
|
||||
if context_memory is None:
|
||||
from GramAddict.core.qdrant_memory import ContextMemoryDB
|
||||
|
||||
self.context_memory = ContextMemoryDB()
|
||||
else:
|
||||
self.context_memory = context_memory
|
||||
|
||||
self._last_click_context: Optional[Dict[str, Any]] = None
|
||||
|
||||
def track_click(self, intent: str, node: SpatialNode, xml_context: str = ""):
|
||||
def track_click(self, intent: str, node: SpatialNode, xml_context: str = "", screen_type: str = "UNKNOWN"):
|
||||
"""Stores the context of a click before it's actually performed."""
|
||||
semantic_string = f"text: '{node.text}', desc: '{node.content_desc}', id: '{node.resource_id}'"
|
||||
|
||||
@@ -82,6 +76,7 @@ class ActionMemory:
|
||||
"node_dict": node.to_dict(),
|
||||
"semantic_string": semantic_string,
|
||||
"xml_context": xml_context,
|
||||
"screen_type": screen_type,
|
||||
}
|
||||
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
|
||||
|
||||
@@ -98,14 +93,8 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
# ── Semantic Mismatch Guard ──
|
||||
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' — "
|
||||
f"clicked element does not match intent: {ctx['semantic_string']}"
|
||||
)
|
||||
self._last_click_context = None
|
||||
return
|
||||
# Zero-Trust FSD: No semantic string mismatch guards here.
|
||||
# If the VLM/Delta verification passed, we trust it and learn.
|
||||
|
||||
logger.info(
|
||||
f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.",
|
||||
@@ -120,6 +109,10 @@ class ActionMemory:
|
||||
self.ui_memory.boost_confidence(ctx["intent"], ctx["xml_context"])
|
||||
else:
|
||||
self.ui_memory.store_memory(ctx["intent"], ctx["xml_context"], ctx["node_dict"])
|
||||
# Boost context confidence
|
||||
screen_type = ctx.get("screen_type", "UNKNOWN")
|
||||
self.context_memory.update_confidence(ctx["intent"], screen_type, delta=0.2)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to confirm click in Qdrant: {e}")
|
||||
|
||||
@@ -140,6 +133,11 @@ class ActionMemory:
|
||||
|
||||
try:
|
||||
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
|
||||
|
||||
# Decay context confidence
|
||||
screen_type = ctx.get("screen_type", "UNKNOWN")
|
||||
self.context_memory.update_confidence(ctx["intent"], screen_type, delta=-0.2)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to decay confidence in Qdrant: {e}")
|
||||
|
||||
@@ -196,22 +194,9 @@ class ActionMemory:
|
||||
state_toggles = ["like", "save", "follow", "heart"]
|
||||
is_toggle = any(t in intent_lower for t in state_toggles)
|
||||
|
||||
# ── P0-1: Structural Resource-ID Bypass Gate ──
|
||||
# If the clicked node was resolved via a structural Resource-ID that
|
||||
# directly matches the toggle intent, VLM verification is SKIPPED.
|
||||
# This eliminates the #1 session failure: VLM hallucinating Follow→Like.
|
||||
if is_toggle and self._last_click_context:
|
||||
clicked_rid = (self._last_click_context.get("node_dict", {}).get("resource_id", "") or "").lower()
|
||||
if clicked_rid:
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in clicked_rid for marker in required_markers):
|
||||
logger.info(
|
||||
f"⚡ [ActionMemory] Structural Resource-ID bypass: '{intent}' matched "
|
||||
f"'{clicked_rid}'. Skipping VLM verification — O(1) trust."
|
||||
)
|
||||
return True
|
||||
break # Only check the first matching intent keyword
|
||||
# P0-1 Bypass Gate removed in FSD architecture.
|
||||
# We NO LONGER bypass VLM verification via string matching.
|
||||
# If confidence is < 0.95, we always do VLM or Delta verification.
|
||||
|
||||
# ── VLM Verification Fallback ──
|
||||
|
||||
@@ -278,14 +263,8 @@ class ActionMemory:
|
||||
logger.error(f"Failed to query VLM for visual verification: {e}")
|
||||
# Fallthrough to structural delta if VLM crashes
|
||||
|
||||
# ── Pre-Structural Semantic Gate ──
|
||||
if is_toggle and self._last_click_context:
|
||||
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
|
||||
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
# Pre-Structural Semantic Gate removed in FSD architecture.
|
||||
# If the delta matches, we trust it. No more static string restrictions.
|
||||
|
||||
# ── Structural Delta Verification ──
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
@@ -315,30 +294,3 @@ class ActionMemory:
|
||||
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
"""Checks if the clicked element semantically matches the toggle intent.
|
||||
|
||||
For toggle intents (follow, like, save), the clicked element MUST contain
|
||||
at least one of the required keywords in its text/desc/id. This prevents
|
||||
photo grid items, captions, and other unrelated elements from being
|
||||
falsely confirmed as successful interactions.
|
||||
|
||||
For non-toggle intents, returns True (no restriction).
|
||||
"""
|
||||
intent_lower = intent.lower()
|
||||
semantic_lower = semantic_string.lower()
|
||||
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in semantic_lower for marker in required_markers):
|
||||
return True
|
||||
logger.debug(
|
||||
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
|
||||
f"{required_markers} but element has: {semantic_string}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Non-toggle intents pass through
|
||||
return True
|
||||
|
||||
@@ -1,90 +1,174 @@
|
||||
import logging
|
||||
from typing import Any, Dict, Set
|
||||
from typing import Any, Dict, FrozenSet, Optional
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════
|
||||
# Categorical Ban Matrix — Structural Impossibility
|
||||
# ══════════════════════════════════════════════════════
|
||||
# These define WHERE each interaction intent is structurally possible.
|
||||
# If a screen is NOT listed for an intent, the action is categorically banned.
|
||||
# This is a WHITELIST: unlisted = impossible. No VLM, no learning, no Qdrant.
|
||||
# This matrix is the Single Source of Truth for structural action plausibility.
|
||||
ALLOWED_SCREENS: Dict[str, FrozenSet[ScreenType]] = {
|
||||
"like": frozenset(
|
||||
{
|
||||
ScreenType.HOME_FEED,
|
||||
ScreenType.POST_DETAIL,
|
||||
ScreenType.REELS_FEED,
|
||||
ScreenType.EXPLORE_GRID, # After opening a post
|
||||
ScreenType.STORY_VIEW, # toolbar_like_button exists on stories
|
||||
}
|
||||
),
|
||||
"comment": frozenset(
|
||||
{
|
||||
ScreenType.HOME_FEED,
|
||||
ScreenType.POST_DETAIL,
|
||||
ScreenType.REELS_FEED,
|
||||
ScreenType.COMMENTS,
|
||||
ScreenType.STORY_VIEW, # reel_viewer_comments_button + message_composer
|
||||
}
|
||||
),
|
||||
"follow": frozenset(
|
||||
{
|
||||
ScreenType.OTHER_PROFILE,
|
||||
ScreenType.FOLLOW_LIST,
|
||||
ScreenType.STORY_VIEW, # reel_header_unconnected_follow_button_stub
|
||||
}
|
||||
),
|
||||
"unfollow": frozenset(
|
||||
{
|
||||
ScreenType.OTHER_PROFILE,
|
||||
ScreenType.FOLLOW_LIST,
|
||||
}
|
||||
),
|
||||
"save": frozenset(
|
||||
{
|
||||
ScreenType.HOME_FEED,
|
||||
ScreenType.POST_DETAIL,
|
||||
ScreenType.REELS_FEED,
|
||||
}
|
||||
),
|
||||
"repost": frozenset(
|
||||
{
|
||||
ScreenType.HOME_FEED,
|
||||
ScreenType.POST_DETAIL,
|
||||
ScreenType.REELS_FEED,
|
||||
}
|
||||
),
|
||||
"share": frozenset(
|
||||
{
|
||||
ScreenType.HOME_FEED,
|
||||
ScreenType.POST_DETAIL,
|
||||
ScreenType.REELS_FEED,
|
||||
ScreenType.STORY_VIEW, # toolbar_reshare_button exists on stories
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
# Intent keywords that trigger the categorical ban check
|
||||
INTERACTION_KEYWORDS = frozenset(ALLOWED_SCREENS.keys())
|
||||
|
||||
|
||||
class ContextGate:
|
||||
"""
|
||||
Validates if an action (intent) is structurally possible on the current screen.
|
||||
This acts as a high-speed circuit breaker before invoking expensive VLM logic.
|
||||
|
||||
Zero-Trust: If the required structural markers aren't in the XML, the action
|
||||
is blocked, even if the LLM/VLM thinks it's possible.
|
||||
Architecture: 2-Layer Cascade
|
||||
─────────────────────────────
|
||||
Layer 0: Categorical Ban Matrix (O(1) dict lookup, zero dependencies)
|
||||
Blocks structurally impossible actions BEFORE any network call.
|
||||
e.g., "like" is impossible on STORY_VIEW — no like button exists.
|
||||
|
||||
Architecture: Uses a whitelist-based Action Compatibility Matrix.
|
||||
Only screens that structurally support an interaction allow it.
|
||||
Layer 1: Qdrant Learned Failures (optional, requires running Qdrant)
|
||||
Blocks actions that have been learned to fail consistently.
|
||||
e.g., "tap follow" on OTHER_PROFILE if that profile's follow button
|
||||
is hidden behind a "Requested" state.
|
||||
"""
|
||||
|
||||
# ── Action Compatibility Matrix (Whitelist) ──
|
||||
# Maps each interaction intent to the set of screens where it's structurally valid.
|
||||
# If an intent is NOT in this map, it's a navigation/system intent and passes through.
|
||||
VALID_SCREENS: Dict[str, Set[ScreenType]] = {
|
||||
"like": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
|
||||
"comment": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.COMMENTS},
|
||||
"follow": {ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST},
|
||||
"unfollow": {ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST},
|
||||
"save": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
|
||||
"repost": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
|
||||
}
|
||||
def __init__(self, context_memory=None):
|
||||
if context_memory is None:
|
||||
try:
|
||||
from GramAddict.core.qdrant_memory import ContextMemoryDB
|
||||
|
||||
# Intent -> List of resource-id fragments that MUST be present on the screen
|
||||
# to even consider performing this action (structural proof).
|
||||
REQUIRED_MARKERS = {
|
||||
"comment": ["comment", "row_feed_button_comment", "shell_comment_button"],
|
||||
"like": ["like", "heart", "row_feed_button_like", "shell_like_button"],
|
||||
"follow": ["follow", "profile_header_follow_button", "button_follow"],
|
||||
"unfollow": ["follow", "profile_header_follow_button", "button_follow"],
|
||||
"save": ["save", "bookmark"],
|
||||
}
|
||||
self.context_memory = ContextMemoryDB()
|
||||
except Exception:
|
||||
self.context_memory = None
|
||||
else:
|
||||
self.context_memory = context_memory
|
||||
|
||||
def is_allowed(self, intent: str, screen_state: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Evaluates the context gate.
|
||||
|
||||
Args:
|
||||
intent: The action name (e.g. 'follow', 'comment')
|
||||
intent: The action name (e.g. 'follow', 'comment', 'tap like button')
|
||||
screen_state: The result of ScreenIdentity.identify()
|
||||
|
||||
Returns:
|
||||
bool: True if the action is plausible, False if it should be blocked.
|
||||
bool: True if the action is plausible (or unknown), False if banned.
|
||||
"""
|
||||
intent_lower = intent.lower()
|
||||
screen_type = screen_state.get("screen_type", ScreenType.UNKNOWN)
|
||||
resource_ids = screen_state.get("resource_ids", set())
|
||||
|
||||
# 1. Action Compatibility Matrix (Whitelist Check)
|
||||
# If the intent matches a known interaction type, check if the screen allows it.
|
||||
for interaction_type, valid_screens in self.VALID_SCREENS.items():
|
||||
if interaction_type in intent_lower:
|
||||
if screen_type not in valid_screens:
|
||||
logger.warning(
|
||||
f"🛡️ [ContextGate] Blocked '{intent}' on {screen_type.name}: "
|
||||
f"Not in valid screens {[s.name for s in valid_screens]}."
|
||||
)
|
||||
return False
|
||||
break # Found the interaction type, don't check others
|
||||
# ── Layer 0: Categorical Ban Matrix (instant, no dependencies) ──
|
||||
matched_keyword = self._extract_interaction_keyword(intent_lower)
|
||||
if matched_keyword is not None and screen_type != ScreenType.UNKNOWN:
|
||||
allowed_screens = ALLOWED_SCREENS[matched_keyword]
|
||||
if screen_type not in allowed_screens:
|
||||
logger.debug(
|
||||
f"🛡️ [ContextGate] BLOCKED '{intent}' on {screen_type.name} — "
|
||||
f"structurally impossible (allowed: {[s.name for s in allowed_screens]})"
|
||||
)
|
||||
return False
|
||||
|
||||
# 2. Structural Marker Verification
|
||||
# Even on a valid screen, the required UI elements must be present.
|
||||
for required_intent, markers in self.REQUIRED_MARKERS.items():
|
||||
if required_intent in intent_lower:
|
||||
has_marker = False
|
||||
for marker in markers:
|
||||
for rid in resource_ids:
|
||||
if marker in rid.lower():
|
||||
has_marker = True
|
||||
break
|
||||
if has_marker:
|
||||
break
|
||||
|
||||
if not has_marker:
|
||||
logger.warning(
|
||||
f"🛡️ [ContextGate] Blocked '{intent}' on {screen_type.name}: "
|
||||
f"No structural markers found {markers}."
|
||||
)
|
||||
return False
|
||||
# ── Layer 1: Qdrant Learned Failures ──
|
||||
if (
|
||||
matched_keyword is not None
|
||||
and screen_type != ScreenType.UNKNOWN
|
||||
and self.context_memory is not None
|
||||
and getattr(self.context_memory, "is_connected", False)
|
||||
):
|
||||
if not self.context_memory.is_allowed(intent_lower, screen_type.name):
|
||||
logger.debug(
|
||||
f"🛡️ [ContextGate] BLOCKED '{intent}' on {screen_type.name} — " f"learned failure from Qdrant"
|
||||
)
|
||||
return False
|
||||
|
||||
# ── Default: Allow (Exploration) ──
|
||||
return True
|
||||
|
||||
def get_valid_screens(self, intent: str) -> Optional[FrozenSet[ScreenType]]:
|
||||
"""
|
||||
Returns the set of screens where an interaction intent is structurally valid.
|
||||
Used by the Planner for auto-routing when the goal can't be achieved on
|
||||
the current screen.
|
||||
|
||||
Returns:
|
||||
FrozenSet[ScreenType] if the intent maps to a known interaction, else None.
|
||||
"""
|
||||
keyword = self._extract_interaction_keyword(intent.lower())
|
||||
if keyword is not None:
|
||||
return ALLOWED_SCREENS[keyword]
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_interaction_keyword(intent_lower: str) -> Optional[str]:
|
||||
"""
|
||||
Extracts the primary interaction keyword from an intent string.
|
||||
Returns None if no interaction keyword is found (i.e., this is a navigation intent).
|
||||
|
||||
Uses word-boundary matching to prevent false positives:
|
||||
- "follow" matches "follow user" but NOT "followers" or "following list"
|
||||
- "like" matches "like post" but NOT "likelihood"
|
||||
"""
|
||||
import re
|
||||
|
||||
for kw in INTERACTION_KEYWORDS:
|
||||
if re.search(rf"\b{kw}\b", intent_lower):
|
||||
return kw
|
||||
return None
|
||||
|
||||
@@ -211,7 +211,7 @@ class IntentResolver:
|
||||
rid = (node.resource_id or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "composer_edittext" in rid or "message…" in text or "message..." in text:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
|
||||
return node
|
||||
|
||||
if "last received message text" in intent_lower or "received message" in intent_lower:
|
||||
@@ -220,7 +220,7 @@ class IntentResolver:
|
||||
if msg_nodes:
|
||||
# The last one in the XML is typically the most recent message at the bottom of the screen
|
||||
latest_msg = msg_nodes[-1]
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
|
||||
return latest_msg
|
||||
|
||||
if "send message button" in intent_lower:
|
||||
@@ -229,24 +229,30 @@ class IntentResolver:
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "send" in rid or "composer_button" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
|
||||
return node
|
||||
|
||||
if "post author username" in intent_lower or "tap post username" in intent_lower:
|
||||
for node in candidates:
|
||||
if "row_feed_photo_profile_imageview" in (node.resource_id or "").lower():
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
|
||||
rid = (node.resource_id or "").lower()
|
||||
if (
|
||||
"row_feed_photo_profile_imageview" in rid
|
||||
or "clips_author_profile_pic" in rid
|
||||
or "reel_viewer_profile_picture" in rid
|
||||
):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
|
||||
return node
|
||||
for node in candidates:
|
||||
if "row_feed_photo_profile_name" in (node.resource_id or "").lower():
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_photo_profile_name" in rid or "clips_author_username" in rid or "reel_viewer_title" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
|
||||
return node
|
||||
|
||||
if "feed post content" in intent_lower or "post media content" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_photo_imageview" in rid or "zoomable_view_container" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
|
||||
return node
|
||||
|
||||
if "comment" in intent_lower and "button" in intent_lower:
|
||||
@@ -266,14 +272,14 @@ class IntentResolver:
|
||||
"row_feed_button_comment" in (node.resource_id or "").lower()
|
||||
or "row_feed_textview_comments" in (node.resource_id or "").lower()
|
||||
):
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
|
||||
return node
|
||||
|
||||
if "like" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_like" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found like button: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found like button: {rid}")
|
||||
return node
|
||||
|
||||
if ("send" in intent_lower or "share" in intent_lower) and "post" in intent_lower and "button" in intent_lower:
|
||||
@@ -281,7 +287,7 @@ class IntentResolver:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row_feed_button_share" in rid or "send post" in desc:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
return node
|
||||
|
||||
if "add to story" in intent_lower:
|
||||
@@ -293,28 +299,51 @@ class IntentResolver:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_share" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found share button: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found share button: {rid}")
|
||||
return node
|
||||
|
||||
if "save" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_save" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found save button: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found save button: {rid}")
|
||||
return node
|
||||
|
||||
if "follow" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "profile_header_follow_button" in rid or "inline_follow_button" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
|
||||
if (
|
||||
"profile_header_follow_button" in rid
|
||||
or "inline_follow_button" in rid
|
||||
or "follow_list_row_large_follow_button" in rid
|
||||
or "row_search_user_follow_button" in rid
|
||||
or "profile_header_user_action_follow_button" in rid
|
||||
):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
|
||||
return node
|
||||
|
||||
if "first post" in intent_lower or "first item" in intent_lower or "first search result" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "grid_card_layout_container" in rid or "image_button" in rid or "row_search_user" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
|
||||
return node
|
||||
|
||||
if "heart" in intent_lower and "notification" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Could be in top bar or bottom bar depending on IG version
|
||||
if "notification" in rid or "newsfeed" in rid or "activity" in desc or "notification" in desc:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found notifications/heart icon: {rid or desc}")
|
||||
return node
|
||||
|
||||
if ("inbox" in intent_lower or "direct message" in intent_lower) and "icon" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "direct_tab" in rid or "inbox_button" in rid or "message" in desc:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found DM/inbox icon: {rid or desc}")
|
||||
return node
|
||||
|
||||
if "story ring" in intent_lower or "story tray" in intent_lower:
|
||||
@@ -354,11 +383,17 @@ class IntentResolver:
|
||||
return story_nodes[0]
|
||||
|
||||
# --- Structural Grid Fast-Paths ---
|
||||
if "first image post in profile grid" in intent_lower or "first post" in intent_lower:
|
||||
if (
|
||||
"first image post in profile grid" in intent_lower
|
||||
or "first post" in intent_lower
|
||||
or "first image" in intent_lower
|
||||
):
|
||||
for node in candidates:
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row 1, column 1" in desc:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found first grid post: {node.resource_id} (desc: '{desc}')")
|
||||
if "row 1" in desc and "column 1" in desc:
|
||||
logger.debug(
|
||||
f"🎯 [Structural Fast-Path] Found first grid post: {node.resource_id} (desc: '{desc}')"
|
||||
)
|
||||
return node
|
||||
|
||||
# --- Navigation Tab Fast-Paths ---
|
||||
@@ -380,7 +415,7 @@ class IntentResolver:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if rid.endswith(f":id/{resource_suffix}"):
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
|
||||
return node
|
||||
|
||||
# Priority 2: Fail Fast
|
||||
@@ -418,7 +453,7 @@ class IntentResolver:
|
||||
|
||||
if semantic_candidates:
|
||||
if len(semantic_candidates) == 1:
|
||||
logger.info(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
|
||||
logger.debug(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
|
||||
return semantic_candidates[0]
|
||||
else:
|
||||
logger.info(
|
||||
@@ -436,11 +471,9 @@ class IntentResolver:
|
||||
if device is not None and (
|
||||
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
|
||||
):
|
||||
print(f"DEBUG_INTENT: Entering Visual Discovery for '{intent_description}'")
|
||||
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
|
||||
return self._visual_discovery(intent_description, candidates, device, screen_height=screen_height)
|
||||
|
||||
print(f"DEBUG_INTENT: Falling back to Text-based VLM for '{intent_description}'")
|
||||
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
|
||||
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure.
|
||||
@@ -659,6 +692,35 @@ class IntentResolver:
|
||||
filtered_candidates.append(node)
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Reply Guard ---
|
||||
# Prevents VLM from clicking the "Reply to story" or "Send message" input field
|
||||
# when looking for general navigation or "next" buttons.
|
||||
if (
|
||||
"reply" not in intent_lower
|
||||
and "message" not in intent_lower
|
||||
and "comment" not in intent_lower
|
||||
and "type" not in intent_lower
|
||||
and "write" not in intent_lower
|
||||
):
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
res_id = (node.resource_id or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
cls_name = (node.class_name or "").lower()
|
||||
if (
|
||||
"reply" in res_id
|
||||
or "message" in res_id
|
||||
or "comment" in res_id
|
||||
or "antworten" in text
|
||||
or "send message" in text
|
||||
or "nachricht" in text
|
||||
or "edittext" in cls_name
|
||||
):
|
||||
logger.debug(f"🛡️ [Reply Guard] Filtered out input/message box: '{node.text}' ({node.resource_id})")
|
||||
else:
|
||||
filtered_candidates.append(node)
|
||||
candidates = filtered_candidates
|
||||
|
||||
try:
|
||||
annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates)
|
||||
except Exception as e:
|
||||
@@ -742,7 +804,6 @@ class IntentResolver:
|
||||
use_local_edge=True,
|
||||
images_b64=[annotated_b64],
|
||||
)
|
||||
print(f"DEBUG_INTENT: VLM RAW RESPONSE for '{intent_description}': {res}")
|
||||
data = json.loads(res)
|
||||
box_idx = self._parse_box_index(data)
|
||||
selected = self._validate_and_get_node(box_idx, box_map)
|
||||
@@ -871,7 +932,6 @@ class IntentResolver:
|
||||
user_prompt=prompt,
|
||||
use_local_edge=True,
|
||||
)
|
||||
print(f"DEBUG_INTENT: TEXT LLM RAW RESPONSE for '{intent_description}': {res}")
|
||||
data = json.loads(res)
|
||||
idx = data.get("selected_index")
|
||||
if idx is not None and 0 <= idx < len(filtered_candidates):
|
||||
|
||||
@@ -156,6 +156,7 @@ class ScreenIdentity:
|
||||
"selected_tab": selected_tab,
|
||||
"context": context,
|
||||
"signature": signature,
|
||||
"resource_ids": resource_ids,
|
||||
}
|
||||
|
||||
def _classify_screen(
|
||||
@@ -177,12 +178,14 @@ class ScreenIdentity:
|
||||
# Priority 1: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
|
||||
# EXCEPTION: If Qdrant has explicitly learned this screen as NORMAL (via LLM unlearning),
|
||||
# we skip the structural check to prevent false-positive infinite loops.
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
browser_markers = ("ig_browser_text_title", "ig_browser_close_button", "ig_chrome_subsection")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
if not is_normal_override and any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
if any(marker in ids_str for marker in browser_markers):
|
||||
if not is_normal_override and any(marker in ids_str for marker in browser_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] In-App Browser detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
@@ -199,8 +202,20 @@ class ScreenIdentity:
|
||||
"profile_header_business_category",
|
||||
)
|
||||
if any(marker in ids for marker in PROFILE_MARKERS):
|
||||
# OWN_PROFILE is confirmed by the bottom tab OR the presence of 'edit' markers
|
||||
if selected_tab == "profile_tab" or "profile_header_edit_profile_button" in ids:
|
||||
# OWN_PROFILE detection priority cascade:
|
||||
# 1. Selected tab == profile_tab (most reliable — structural)
|
||||
# 2. Edit profile button present (structural)
|
||||
# 3. Bot username found in visible text (semantic fallback)
|
||||
is_own = False
|
||||
if selected_tab == "profile_tab":
|
||||
is_own = True
|
||||
elif "profile_header_edit_profile_button" in ids:
|
||||
is_own = True
|
||||
elif text_lower:
|
||||
if self.bot_username and self.bot_username in text_lower:
|
||||
is_own = True
|
||||
|
||||
if is_own:
|
||||
return ScreenType.OWN_PROFILE
|
||||
return ScreenType.OTHER_PROFILE
|
||||
|
||||
|
||||
@@ -29,7 +29,10 @@ class QdrantBase:
|
||||
|
||||
try:
|
||||
qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344")
|
||||
self.client = QdrantClient(url=qdrant_url, timeout=10.0)
|
||||
if qdrant_url == ":memory:":
|
||||
self.client = QdrantClient(location=":memory:")
|
||||
else:
|
||||
self.client = QdrantClient(url=qdrant_url, timeout=10.0)
|
||||
|
||||
if self.client:
|
||||
if self.client.collection_exists(collection_name):
|
||||
@@ -208,7 +211,12 @@ class QdrantBase:
|
||||
try:
|
||||
res = self.client.retrieve(collection_name=self.collection_name, ids=[point_id])
|
||||
if res:
|
||||
self.client.delete(collection_name=self.collection_name, points_selector=[point_id])
|
||||
from qdrant_client.models import PointIdsList
|
||||
|
||||
self.client.delete(
|
||||
collection_name=self.collection_name,
|
||||
points_selector=PointIdsList(points=[point_id]),
|
||||
)
|
||||
logger.info(
|
||||
f"🗑️ [Qdrant] Purged poisoned memory vector from {self.collection_name} (UUID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[31m"},
|
||||
@@ -1368,6 +1376,156 @@ class ParasocialCRMDB(QdrantBase):
|
||||
return "\n".join(context_parts)
|
||||
|
||||
|
||||
class FailureJournalDB(QdrantBase):
|
||||
"""
|
||||
P1-1: Crash Black Box.
|
||||
Stores exact state context and intents that led to softlocks, crashes, or "LIE DETECTED"
|
||||
phantom execution. Allows the system to learn what NOT to do.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(collection_name="gramaddict_failure_journal_v1")
|
||||
|
||||
def record_failure(self, screen_state: str, intent: str, error_msg: str):
|
||||
if not self.is_connected:
|
||||
return
|
||||
|
||||
failure_key = f"{screen_state}_{intent}"
|
||||
vector = self._get_embedding(failure_key)
|
||||
if not vector:
|
||||
return
|
||||
|
||||
payload = {
|
||||
"screen_state": screen_state,
|
||||
"intent": intent,
|
||||
"error_msg": error_msg,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
||||
self.upsert_point(
|
||||
seed_string=failure_key,
|
||||
vector=vector,
|
||||
payload=payload,
|
||||
log_success=f"📓 [FailureJournal] Recorded critical failure: {intent} on {screen_state}",
|
||||
)
|
||||
|
||||
def is_known_failure(self, screen_state: str, intent: str, threshold: float = 0.95) -> bool:
|
||||
if not self.is_connected:
|
||||
return False
|
||||
|
||||
failure_key = f"{screen_state}_{intent}"
|
||||
vector = self._get_embedding(failure_key)
|
||||
if not vector:
|
||||
return False
|
||||
|
||||
try:
|
||||
results = self.client.query_points(
|
||||
collection_name=self.collection_name,
|
||||
query=vector,
|
||||
limit=1,
|
||||
).points
|
||||
|
||||
if results and results[0].score >= threshold:
|
||||
logger.warning(
|
||||
f"📓 [FailureJournal] Circuit Breaker: Preventing known fatal action '{intent}' on '{screen_state}'"
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class ContextMemoryDB(QdrantBase):
|
||||
"""
|
||||
Learns which intents are possible on which screens.
|
||||
Replaces ContextGate's hardcoded VALID_SCREENS whitelist.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(collection_name="gramaddict_context_memory_v1", vector_size=128)
|
||||
|
||||
def _get_key(self, intent: str, screen_type: str) -> str:
|
||||
return f"{intent}_{screen_type}"
|
||||
|
||||
def update_confidence(self, intent: str, screen_type: str, delta: float):
|
||||
if not self.is_connected:
|
||||
return
|
||||
|
||||
key = self._get_key(intent, screen_type)
|
||||
try:
|
||||
# Generate a consistent ID
|
||||
point_id = self.generate_uuid(key)
|
||||
points = self.client.retrieve(
|
||||
collection_name=self.collection_name, ids=[point_id], with_payload=True, with_vectors=False
|
||||
)
|
||||
|
||||
# Default neutral confidence for new state-intent pairs is 0.5
|
||||
current_conf = 0.5
|
||||
if points:
|
||||
current_conf = points[0].payload.get("confidence", 0.5)
|
||||
|
||||
new_conf = max(0.0, min(1.0, current_conf + delta))
|
||||
|
||||
# Using zero-vector for fast KV-like lookup, since we rely entirely on exact point_id match
|
||||
vector = [0.0] * self._vector_size
|
||||
|
||||
from qdrant_client.models import PointStruct
|
||||
|
||||
self.client.upsert(
|
||||
collection_name=self.collection_name,
|
||||
points=[
|
||||
PointStruct(
|
||||
id=point_id,
|
||||
vector=vector,
|
||||
payload={
|
||||
"intent": intent,
|
||||
"screen_type": screen_type,
|
||||
"confidence": new_conf,
|
||||
"updated_at": time.time(),
|
||||
},
|
||||
)
|
||||
],
|
||||
wait=True,
|
||||
)
|
||||
color = "\x1b[32m" if delta > 0 else "\x1b[31m"
|
||||
symbol = "📈" if delta > 0 else "📉"
|
||||
logger.info(
|
||||
f"{symbol} [ContextMemory] Confidence for '{intent}' on '{screen_type}' adjusted to {new_conf:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"ContextMemory error: {e}")
|
||||
|
||||
def is_allowed(self, intent: str, screen_type: str) -> bool:
|
||||
"""
|
||||
Exploration vs Exploitation:
|
||||
If we don't know (no entry or neutral confidence), allow it!
|
||||
Only block if we have learned it fails consistently (< 0.2).
|
||||
"""
|
||||
if not self.is_connected:
|
||||
return True # Fail-open for exploration
|
||||
|
||||
key = self._get_key(intent, screen_type)
|
||||
try:
|
||||
point_id = self.generate_uuid(key)
|
||||
points = self.client.retrieve(
|
||||
collection_name=self.collection_name, ids=[point_id], with_payload=True, with_vectors=False
|
||||
)
|
||||
if points:
|
||||
conf = points[0].payload.get("confidence", 0.5)
|
||||
if conf < 0.2:
|
||||
logger.warning(
|
||||
f"🛡️ [ContextMemory] Circuit Breaker: Blocked '{intent}' on '{screen_type}' "
|
||||
f"(learned confidence {conf:.2f} < 0.2)"
|
||||
)
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True # Allow exploration
|
||||
|
||||
|
||||
def wipe_all_ai_caches():
|
||||
"""
|
||||
Wipes ALL global (non-user-specific) Qdrant AI caches.
|
||||
|
||||
@@ -29,6 +29,7 @@ class SituationType(Enum):
|
||||
OBSTACLE_MODAL = "obstacle_modal"
|
||||
OBSTACLE_FOREIGN_APP = "obstacle_foreign_app"
|
||||
OBSTACLE_SYSTEM = "obstacle_system"
|
||||
OBSTACLE_KEYBOARD = "obstacle_keyboard"
|
||||
DANGER_ACTION_BLOCKED = "danger_action_blocked"
|
||||
|
||||
|
||||
@@ -337,6 +338,18 @@ class SituationalAwarenessEngine:
|
||||
logger.info("📱 [SAE Perceive] System permission dialog explicitly detected.")
|
||||
return SituationType.OBSTACLE_SYSTEM
|
||||
|
||||
# ── Keyboard Detection (Fast Path) ──
|
||||
keyboard_pkgs = {
|
||||
"com.google.android.inputmethod.latin",
|
||||
"com.samsung.android.honeyboard",
|
||||
"com.sec.android.inputmethod",
|
||||
"com.touchtype.swiftkey",
|
||||
"com.apple.android.inputmethod",
|
||||
}
|
||||
if any(pkg in keyboard_pkgs for pkg in packages):
|
||||
logger.info("📱 [SAE Perceive] On-screen Keyboard explicitly detected. Treating as obstacle.")
|
||||
return SituationType.OBSTACLE_KEYBOARD
|
||||
|
||||
# ── Foreign Environment Detection (package-based) ──
|
||||
# If the main app package is completely absent from the UI hierarchy,
|
||||
# OR if there's a dominant foreign package and no app package, we might have lost the app.
|
||||
|
||||
@@ -118,7 +118,11 @@ class TelepathicEngine:
|
||||
|
||||
# 4. Track action
|
||||
if track:
|
||||
self._memory.track_click(intent_description, best_node, xml_string)
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity
|
||||
|
||||
screen_state = ScreenIdentity("").identify(xml_string)
|
||||
screen_type = screen_state.get("screen_type").name if screen_state.get("screen_type") else "UNKNOWN"
|
||||
self._memory.track_click(intent_description, best_node, xml_string, screen_type=screen_type)
|
||||
|
||||
# Translate to old GramAddict dict format for backward compatibility
|
||||
return self._translate_node(best_node)
|
||||
|
||||
Reference in New Issue
Block a user