feat(navigation): harden obstacle guard and intent resolver against keyboard hallucinations

This commit is contained in:
2026-05-05 10:52:38 +02:00
parent 39185593dd
commit 5ada31c77e
40 changed files with 950 additions and 1067 deletions

View File

@@ -157,7 +157,7 @@ class PluginRegistry:
self._plugins.append(plugin)
self._sorted = False
logger.info(f"🧩 [Plugin] Registered: {plugin.name} (priority={plugin.priority})")
logger.debug(f"🧩 [Plugin] Registered: {plugin.name} (priority={plugin.priority})")
def unregister(self, name: str):
"""Remove a plugin by name."""
@@ -206,6 +206,10 @@ class PluginRegistry:
logger.debug(f"🧩 [PluginRegistry] TRACE: Calling execute() on {plugin.name}")
result = plugin.execute(ctx)
results.append(result)
if result.executed:
logger.debug(
f"🧩 [PluginRegistry] Plugin {plugin.name} executed successfully. Metadata: {result.metadata}"
)
if (plugin.exclusive and result.executed) or result.should_skip:
logger.debug(

View File

@@ -32,18 +32,16 @@ class CommentPlugin(BehaviorPlugin):
if ctx.session_state.check_limit(SessionState.Limit.COMMENTS):
return False
# Safety Guard: Do not comment on stories or grids
xml_lower = (ctx.context_xml or "").lower()
STORY_MARKERS = (
"reel_viewer_media_layout",
"reel_viewer_header",
"reel_viewer_progress_bar",
"reel_viewer_root",
)
if any(marker in xml_lower for marker in STORY_MARKERS):
return False
# ── STRUCTURAL GUARD ──
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
if "explore_action_bar" in xml_lower or "profile_tabs_container" in xml_lower:
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
if screen_type not in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED):
return False
config = self.get_config(ctx)

View File

@@ -30,6 +30,17 @@ class DarwinDwellPlugin(BehaviorPlugin):
if not getattr(self, "_enabled", True):
return False
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
if screen_type not in valid_screens:
return False
config = self.get_config(ctx)
percentage = float(config.get("percentage", 100))
return random.random() < (percentage / 100.0)

View File

@@ -35,6 +35,18 @@ class FollowPlugin(BehaviorPlugin):
if ctx.session_state.check_limit(SessionState.Limit.FOLLOWS):
return False
# ── STRUCTURAL GUARD ──
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
if screen_type not in (ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST):
return False
# Probability gate
if random.random() >= follow_pct:
return False
@@ -49,8 +61,8 @@ class FollowPlugin(BehaviorPlugin):
nav_graph = QNavGraph(ctx.device)
if nav_graph.do("tap 'Follow' button") or nav_graph.do("tap 'Following' button"):
logger.info(f"🤝 [Follow] Followed @{ctx.username}")
if nav_graph.do("tap 'Follow' or 'Following' button"):
logger.info(f"🤝 [Follow] Toggled Follow/Following state for @{ctx.username}")
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
# Buffer for follow animations to close

View File

@@ -38,14 +38,15 @@ class GridLikePlugin(BehaviorPlugin):
return False
# ── STRUCTURAL GUARD ──
nav_graph = ctx.cognitive_stack.get("nav_graph")
if nav_graph and nav_graph.current_state != "ProfileView":
return False
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
# Fallback XML check
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
xml_lower = xml.lower()
if "followers" not in xml_lower and "beiträge" not in xml_lower and "posts" not in xml_lower:
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
if screen_type not in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE, ScreenType.EXPLORE_GRID):
return False
# Probability gate
@@ -75,7 +76,20 @@ class GridLikePlugin(BehaviorPlugin):
nav_graph = QNavGraph(ctx.device)
if not nav_graph.do("tap first image post in profile grid"):
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
nav_action = (
"tap first image in explore grid"
if screen_type == ScreenType.EXPLORE_GRID
else "tap first image post in profile grid"
)
if not nav_graph.do(nav_action):
return BehaviorResult(executed=False, metadata={"reason": "grid_nav_failed"})
if not wait_for_post_loaded(ctx.device, timeout=5, nav_graph=nav_graph):

View File

@@ -39,6 +39,18 @@ class LikePlugin(BehaviorPlugin):
if likes_pct <= 0:
return False
# ── STRUCTURAL GUARD ──
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
if screen_type not in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED):
return False
# Probability gate
if random.random() >= likes_pct:
return False

View File

@@ -56,6 +56,13 @@ class ObstacleGuardPlugin(BehaviorPlugin):
sleep(1.5 * ctx.sleep_mod)
return BehaviorResult(executed=True, should_skip=True)
# ── On-Screen Keyboard (e.g. hallucinated click on comment field) ──
if situation == SituationType.OBSTACLE_KEYBOARD:
logger.warning("⚠️ [ObstacleGuard] On-screen Keyboard is open. Pressing BACK to dismiss...")
ctx.device.press("back")
sleep(1.0 * ctx.sleep_mod)
return BehaviorResult(executed=True, should_skip=True)
# ── Instagram Modal / Overlay (survey, "Not Now" prompt, creation flow) ──
if situation == SituationType.OBSTACLE_MODAL:
if misses >= 2:

View File

@@ -31,7 +31,18 @@ class PostInteractionPlugin(BehaviorPlugin):
return True # Ends the behavior chain for this post
def can_activate(self, ctx: BehaviorContext) -> bool:
return getattr(self, "_enabled", True)
if not getattr(self, "_enabled", True):
return False
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
return screen_type in valid_screens
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
logger.info("🏁 [PostInteraction] Interactions complete. Moving to next post...")

View File

@@ -31,12 +31,24 @@ class ProfileVisitPlugin(BehaviorPlugin):
if not getattr(self, "_enabled", True):
return False
# 1. Guard against recursive calls or being already on profile
# 1. Screen Guard: Only activate on feed screens
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
valid_screens = [ScreenType.HOME_FEED, ScreenType.EXPLORE_GRID, ScreenType.REELS_FEED]
if screen_type not in valid_screens:
return False
# 2. Guard against recursive calls or being already on profile
nav_graph = ctx.cognitive_stack.get("nav_graph")
if nav_graph and nav_graph.current_state == "ProfileView":
return False
# 2. Probability gate
# 3. Probability gate
config = self.get_config(ctx)
visit_pct = float(config.get("percentage", getattr(ctx.configs.args, "profile_visit_percentage", 30))) / 100.0

View File

@@ -30,6 +30,18 @@ class RepostPlugin(BehaviorPlugin):
if not getattr(self, "_enabled", True):
return False
# 1. Screen Guard: Only activate on post-containing screens
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED]
if screen_type not in valid_screens:
return False
config = self.get_config(ctx)
repost_pct = float(config.get("percentage", getattr(ctx.configs.args, "repost_percentage", 20))) / 100.0

View File

@@ -27,7 +27,17 @@ class ResonanceEvaluatorPlugin(BehaviorPlugin):
return 80
def can_activate(self, ctx: BehaviorContext) -> bool:
return getattr(self, "_enabled", True)
if not getattr(self, "_enabled", True):
return False
screen_type = ctx.shared_state.get("current_screen_type")
if not screen_type and ctx.context_xml:
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_type = ScreenIdentity(getattr(ctx, "username", "")).identify(ctx.context_xml).get("screen_type")
from GramAddict.core.perception.screen_identity import ScreenType
valid_screens = [ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.REELS_FEED, ScreenType.STORY_VIEW]
return screen_type in valid_screens
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
resonance = ctx.cognitive_stack.get("resonance")

View File

@@ -1,5 +1,6 @@
import logging
import random
import re
from time import sleep
from GramAddict.core.behaviors import BehaviorContext, BehaviorPlugin, BehaviorResult
@@ -96,8 +97,39 @@ class StoryViewPlugin(BehaviorPlugin):
sleep(random.uniform(2.0, 5.0) * ctx.sleep_mod)
if i < count - 1:
humanized_click(ctx.device, int(w * 0.9), int(h * 0.5), sleep_mod=ctx.sleep_mod)
# Atomic state validation after click
xml_dump = ctx.device.dump_hierarchy()
if not xml_dump:
continue
packages = set(re.findall(r'package="([^"]+)"', xml_dump))
app_id = getattr(ctx.device, "app_id", "com.instagram.android")
if packages and app_id not in packages:
logger.error(
f"🚨 [StoryView] FOREIGN APP DETECTED! Packages: {packages}. "
f"A link likely opened an external app. Aborting loop."
)
ctx.device.press("back")
sleep(1.5)
break
ctx.device.press("back")
sleep(random.uniform(1.0, 2.0) * ctx.sleep_mod)
# Post-interaction verification: verify we successfully exited the story overlay
for attempt in range(3):
xml_dump = ctx.device.dump_hierarchy()
if not xml_dump:
break
xml_lower = xml_dump.lower()
if "com.instagram.android" not in xml_dump or (
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
):
# Successfully back to a main view or outside instagram
break
logger.warning(
f"⚠️ [StoryView] Still trapped in story/overlay after back press (attempt {attempt+1}). Pressing back again."
)
ctx.device.press("back")
sleep(1.5)
return BehaviorResult(executed=True, interactions=count, metadata={"stories_viewed": count})

View File

@@ -621,6 +621,24 @@ def start_bot(**kwargs):
break
logger.info(f"Session complete. Boredom: {dopamine.boredom:.1f}%. Sleeping before next iteration...")
# ── P1-3: Wire EvolutionEngine ──
try:
from GramAddict.core.evolution_engine import EvolutionEngine, SessionResult
evolution_engine = EvolutionEngine.get_instance(username)
session_result = SessionResult(
follows_gained=sum(session_state.totalFollowed.values()),
likes_given=session_state.totalLikes,
stories_viewed=getattr(session_state, "totalWatched", 0),
blocks_received=getattr(session_state, "totalBlocks", 0),
duration_minutes=(datetime.now() - session_state.startTime).total_seconds() / 60.0,
profiles_scraped=getattr(session_state, "totalScraped", 0),
)
evolution_engine.evolve(session_result)
except Exception as e:
logger.error(f"⚠️ Failed to run EvolutionEngine: {e}")
close_instagram(device)
random_sleep(30, 60)
@@ -872,7 +890,23 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
)
device.press("back")
sleep(1.5)
return "CONTEXT_LOST"
# Recover gracefully instead of forcing a nuclear app restart
for attempt in range(3):
xml_dump = device.dump_hierarchy()
if not xml_dump:
break
xml_lower = xml_dump.lower()
if app_id in xml_dump and (
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
):
break
logger.warning(
f"⚠️ [StoriesFeed] Still trapped after back press (attempt {attempt+1}). Pressing back again."
)
device.press("back")
sleep(1.5)
return "FEED_EXHAUSTED"
if getattr(configs.args, "ignore_close_friends", False):
if "enge freunde" in xml_dump.lower() or "close friend" in xml_dump.lower():
@@ -892,6 +926,24 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
logger.info("🎬 [StoriesFeed] Session completed naturally.")
device.press("back")
sleep(1.5)
# ── Strict Navigation-Verification ──
for attempt in range(3):
xml_dump = device.dump_hierarchy()
if not xml_dump:
break
xml_lower = xml_dump.lower()
if "com.instagram.android" in xml_dump and (
"row 1, column 1" in xml_lower or "tab" in xml_lower or "home" in xml_lower or "search" in xml_lower
):
break
logger.warning(
f"⚠️ [StoriesFeed] Still trapped in story/overlay after back press (attempt {attempt+1}). Pressing back again."
)
device.press("back")
sleep(1.5)
return "FEED_EXHAUSTED"

View File

@@ -106,7 +106,7 @@ class DopamineEngine:
return True
# True if we have scrolled too long or hit absolute burnout
return (time.time() - self.session_start) > self.session_limit_seconds or self.boredom >= 100.0
return (time.time() - self.session_start) >= self.session_limit_seconds or self.boredom >= 100.0
def get_pacing_modifier(self, base_score: float):
"""

View File

@@ -375,25 +375,6 @@ class GoalExecutor:
self._get_sae().ensure_clear_screen(max_attempts=3)
return False
# ── Pre-Click Semantic Match Guard ──
# For toggle intents (follow/like/save), verify the selected node
# semantically matches the intent BEFORE clicking. This prevents
# VLM hallucinations from clicking photo grid items when looking
# for follow buttons.
from GramAddict.core.perception.action_memory import _intent_matches_node
node_semantic = (
f"text: '{best_node.get('text', '')}', "
f"desc: '{best_node.get('description', '')}', "
f"id: '{best_node.get('id', '')}'"
)
if not _intent_matches_node(action, node_semantic):
logger.warning(
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
f"Node: {node_semantic}"
)
return False
# Execute click
self.device.click(obj=best_node)
import random

View File

@@ -53,16 +53,16 @@ def ask_brain_for_action(
)
if response:
result = response if isinstance(response, str) else response.get("response", "")
result = result.strip().strip("'\"")
result = result.strip().strip("'\"").rstrip(".")
# 1. Exact match check (ideal case)
for act in available_actions:
if act.lower() == result.lower():
return act
# 2. Strict line-by-line check (often the model outputs the action on the last line)
for line in reversed(result.splitlines()):
line = line.strip().strip("'\"")
line = line.strip().strip("'\"").rstrip(".")
for act in available_actions:
if act.lower() == line.lower():
return act
@@ -75,12 +75,14 @@ def ask_brain_for_action(
if idx > best_idx:
best_idx = idx
best_act = act
if best_act:
logger.warning(f"🧠 [Brain] Extracted action '{best_act}' from verbose LLM output.")
return best_act
logger.warning(f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back.")
logger.warning(
f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back."
)
except Exception as e:
logger.debug(f"🧠 [Brain] Error querying LLM: {e}")

View File

@@ -186,6 +186,11 @@ class NavigationKnowledge:
def is_trap(self, screen_type: ScreenType, action: str) -> bool:
"""Check if an action on this screen is a known trap."""
from GramAddict.core.screen_topology import ScreenTopology
if ScreenTopology.is_structural_action(screen_type, action):
return False # Structural actions can NEVER be traps
trap_key = f"{screen_type.name}_{action}"
if trap_key in self._learned_traps:
return True

View File

@@ -178,6 +178,36 @@ class GoalPlanner:
f"🛡️ [HD Map] Route action '{next_action}' already explored and failed. Skipping HD Map."
)
# ── 2.5. ContextGate Feedback Loop ──
# Preempt the brain from hallucinating banned interaction intents.
from GramAddict.core.perception.context_gate import ContextGate
cg = ContextGate()
valid_screens = cg.get_valid_screens(goal)
if valid_screens is not None and screen_type not in valid_screens:
logger.warning(
f"🛡️ [Planner Feedback] Goal '{goal}' is structurally banned on {screen_type.name} by ContextGate."
)
# We are trapped from doing the goal here. Must navigate to one of the valid screens.
best_route = None
for vs in valid_screens:
r = ScreenTopology.find_route(screen_type, vs, avoid_actions=avoid_actions)
if r and (not best_route or len(r) < len(best_route)):
best_route = r
if best_route:
next_action, next_screen = best_route[0]
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
logger.info(
f"🗺️ [Planner Feedback] Auto-routing to {best_route[-1][1].name} via '{next_action}'"
)
return next_action
# If no route found, force back-tracking or skip brain to avoid hallucination.
if "press back" in available:
return "press back"
return None
# ── 3. Brain-Driven Decision Making (Fallback / Discovery) ──
# For non-navigation goals or when the HD Map is incomplete.
from GramAddict.core.navigation.brain import ask_brain_for_action

View File

@@ -39,21 +39,8 @@ def _parse_yes_no(response: str) -> Optional[bool]:
return None
# ═══════════════════════════════════════════════════════
# Semantic Match Keywords — SSOT for intent → element validation
# ═══════════════════════════════════════════════════════
# Maps toggle-intent keywords to required element markers.
# If the intent contains the key, the clicked element MUST
# contain at least one of the corresponding markers in its
# text, content_desc, or resource_id.
# ZERO MAINTENANCE: Only English words and resource_id fragments allowed.
# No localized strings — the bot must work on any device language.
TOGGLE_INTENT_MARKERS = {
"follow": ["follow", "button_follow"],
"like": ["like", "heart", "button_like"],
"save": ["save", "saved", "bookmark"],
}
# FSD Architecture: No static string dictionaries.
# The bot relies 100% on learned confidence and VLM/Delta verification.
class ActionMemory:
@@ -62,8 +49,8 @@ class ActionMemory:
Decouples the memory layer from the core parsing engine.
"""
def __init__(self, ui_memory=None):
# We optionally inject UIMemoryDB to decouple tests
def __init__(self, ui_memory=None, context_memory=None):
# We optionally inject UIMemoryDB and ContextMemoryDB to decouple tests
if ui_memory is None:
from GramAddict.core.qdrant_memory import UIMemoryDB
@@ -71,9 +58,16 @@ class ActionMemory:
else:
self.ui_memory = ui_memory
if context_memory is None:
from GramAddict.core.qdrant_memory import ContextMemoryDB
self.context_memory = ContextMemoryDB()
else:
self.context_memory = context_memory
self._last_click_context: Optional[Dict[str, Any]] = None
def track_click(self, intent: str, node: SpatialNode, xml_context: str = ""):
def track_click(self, intent: str, node: SpatialNode, xml_context: str = "", screen_type: str = "UNKNOWN"):
"""Stores the context of a click before it's actually performed."""
semantic_string = f"text: '{node.text}', desc: '{node.content_desc}', id: '{node.resource_id}'"
@@ -82,6 +76,7 @@ class ActionMemory:
"node_dict": node.to_dict(),
"semantic_string": semantic_string,
"xml_context": xml_context,
"screen_type": screen_type,
}
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
@@ -98,14 +93,8 @@ class ActionMemory:
if intent and ctx["intent"] != intent:
return
# ── Semantic Mismatch Guard ──
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
logger.warning(
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}'"
f"clicked element does not match intent: {ctx['semantic_string']}"
)
self._last_click_context = None
return
# Zero-Trust FSD: No semantic string mismatch guards here.
# If the VLM/Delta verification passed, we trust it and learn.
logger.info(
f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.",
@@ -120,6 +109,10 @@ class ActionMemory:
self.ui_memory.boost_confidence(ctx["intent"], ctx["xml_context"])
else:
self.ui_memory.store_memory(ctx["intent"], ctx["xml_context"], ctx["node_dict"])
# Boost context confidence
screen_type = ctx.get("screen_type", "UNKNOWN")
self.context_memory.update_confidence(ctx["intent"], screen_type, delta=0.2)
except Exception as e:
logger.warning(f"Failed to confirm click in Qdrant: {e}")
@@ -140,6 +133,11 @@ class ActionMemory:
try:
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
# Decay context confidence
screen_type = ctx.get("screen_type", "UNKNOWN")
self.context_memory.update_confidence(ctx["intent"], screen_type, delta=-0.2)
except Exception as e:
logger.warning(f"Failed to decay confidence in Qdrant: {e}")
@@ -196,22 +194,9 @@ class ActionMemory:
state_toggles = ["like", "save", "follow", "heart"]
is_toggle = any(t in intent_lower for t in state_toggles)
# ── P0-1: Structural Resource-ID Bypass Gate ──
# If the clicked node was resolved via a structural Resource-ID that
# directly matches the toggle intent, VLM verification is SKIPPED.
# This eliminates the #1 session failure: VLM hallucinating Follow→Like.
if is_toggle and self._last_click_context:
clicked_rid = (self._last_click_context.get("node_dict", {}).get("resource_id", "") or "").lower()
if clicked_rid:
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
if intent_keyword in intent_lower:
if any(marker in clicked_rid for marker in required_markers):
logger.info(
f"⚡ [ActionMemory] Structural Resource-ID bypass: '{intent}' matched "
f"'{clicked_rid}'. Skipping VLM verification — O(1) trust."
)
return True
break # Only check the first matching intent keyword
# P0-1 Bypass Gate removed in FSD architecture.
# We NO LONGER bypass VLM verification via string matching.
# If confidence is < 0.95, we always do VLM or Delta verification.
# ── VLM Verification Fallback ──
@@ -278,14 +263,8 @@ class ActionMemory:
logger.error(f"Failed to query VLM for visual verification: {e}")
# Fallthrough to structural delta if VLM crashes
# ── Pre-Structural Semantic Gate ──
if is_toggle and self._last_click_context:
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
logger.warning(
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
)
return False
# Pre-Structural Semantic Gate removed in FSD architecture.
# If the delta matches, we trust it. No more static string restrictions.
# ── Structural Delta Verification ──
diff = abs(len(pre_click_xml) - len(post_click_xml))
@@ -315,30 +294,3 @@ class ActionMemory:
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
)
return False
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
"""Checks if the clicked element semantically matches the toggle intent.
For toggle intents (follow, like, save), the clicked element MUST contain
at least one of the required keywords in its text/desc/id. This prevents
photo grid items, captions, and other unrelated elements from being
falsely confirmed as successful interactions.
For non-toggle intents, returns True (no restriction).
"""
intent_lower = intent.lower()
semantic_lower = semantic_string.lower()
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
if intent_keyword in intent_lower:
if any(marker in semantic_lower for marker in required_markers):
return True
logger.debug(
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
f"{required_markers} but element has: {semantic_string}"
)
return False
# Non-toggle intents pass through
return True

View File

@@ -1,90 +1,174 @@
import logging
from typing import Any, Dict, Set
from typing import Any, Dict, FrozenSet, Optional
from GramAddict.core.perception.screen_identity import ScreenType
logger = logging.getLogger(__name__)
# ══════════════════════════════════════════════════════
# Categorical Ban Matrix — Structural Impossibility
# ══════════════════════════════════════════════════════
# These define WHERE each interaction intent is structurally possible.
# If a screen is NOT listed for an intent, the action is categorically banned.
# This is a WHITELIST: unlisted = impossible. No VLM, no learning, no Qdrant.
# This matrix is the Single Source of Truth for structural action plausibility.
ALLOWED_SCREENS: Dict[str, FrozenSet[ScreenType]] = {
"like": frozenset(
{
ScreenType.HOME_FEED,
ScreenType.POST_DETAIL,
ScreenType.REELS_FEED,
ScreenType.EXPLORE_GRID, # After opening a post
ScreenType.STORY_VIEW, # toolbar_like_button exists on stories
}
),
"comment": frozenset(
{
ScreenType.HOME_FEED,
ScreenType.POST_DETAIL,
ScreenType.REELS_FEED,
ScreenType.COMMENTS,
ScreenType.STORY_VIEW, # reel_viewer_comments_button + message_composer
}
),
"follow": frozenset(
{
ScreenType.OTHER_PROFILE,
ScreenType.FOLLOW_LIST,
ScreenType.STORY_VIEW, # reel_header_unconnected_follow_button_stub
}
),
"unfollow": frozenset(
{
ScreenType.OTHER_PROFILE,
ScreenType.FOLLOW_LIST,
}
),
"save": frozenset(
{
ScreenType.HOME_FEED,
ScreenType.POST_DETAIL,
ScreenType.REELS_FEED,
}
),
"repost": frozenset(
{
ScreenType.HOME_FEED,
ScreenType.POST_DETAIL,
ScreenType.REELS_FEED,
}
),
"share": frozenset(
{
ScreenType.HOME_FEED,
ScreenType.POST_DETAIL,
ScreenType.REELS_FEED,
ScreenType.STORY_VIEW, # toolbar_reshare_button exists on stories
}
),
}
# Intent keywords that trigger the categorical ban check
INTERACTION_KEYWORDS = frozenset(ALLOWED_SCREENS.keys())
class ContextGate:
"""
Validates if an action (intent) is structurally possible on the current screen.
This acts as a high-speed circuit breaker before invoking expensive VLM logic.
Zero-Trust: If the required structural markers aren't in the XML, the action
is blocked, even if the LLM/VLM thinks it's possible.
Architecture: 2-Layer Cascade
─────────────────────────────
Layer 0: Categorical Ban Matrix (O(1) dict lookup, zero dependencies)
Blocks structurally impossible actions BEFORE any network call.
e.g., "like" is impossible on STORY_VIEW — no like button exists.
Architecture: Uses a whitelist-based Action Compatibility Matrix.
Only screens that structurally support an interaction allow it.
Layer 1: Qdrant Learned Failures (optional, requires running Qdrant)
Blocks actions that have been learned to fail consistently.
e.g., "tap follow" on OTHER_PROFILE if that profile's follow button
is hidden behind a "Requested" state.
"""
# ── Action Compatibility Matrix (Whitelist) ──
# Maps each interaction intent to the set of screens where it's structurally valid.
# If an intent is NOT in this map, it's a navigation/system intent and passes through.
VALID_SCREENS: Dict[str, Set[ScreenType]] = {
"like": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
"comment": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL, ScreenType.COMMENTS},
"follow": {ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST},
"unfollow": {ScreenType.OTHER_PROFILE, ScreenType.FOLLOW_LIST},
"save": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
"repost": {ScreenType.HOME_FEED, ScreenType.POST_DETAIL},
}
def __init__(self, context_memory=None):
if context_memory is None:
try:
from GramAddict.core.qdrant_memory import ContextMemoryDB
# Intent -> List of resource-id fragments that MUST be present on the screen
# to even consider performing this action (structural proof).
REQUIRED_MARKERS = {
"comment": ["comment", "row_feed_button_comment", "shell_comment_button"],
"like": ["like", "heart", "row_feed_button_like", "shell_like_button"],
"follow": ["follow", "profile_header_follow_button", "button_follow"],
"unfollow": ["follow", "profile_header_follow_button", "button_follow"],
"save": ["save", "bookmark"],
}
self.context_memory = ContextMemoryDB()
except Exception:
self.context_memory = None
else:
self.context_memory = context_memory
def is_allowed(self, intent: str, screen_state: Dict[str, Any]) -> bool:
"""
Evaluates the context gate.
Args:
intent: The action name (e.g. 'follow', 'comment')
intent: The action name (e.g. 'follow', 'comment', 'tap like button')
screen_state: The result of ScreenIdentity.identify()
Returns:
bool: True if the action is plausible, False if it should be blocked.
bool: True if the action is plausible (or unknown), False if banned.
"""
intent_lower = intent.lower()
screen_type = screen_state.get("screen_type", ScreenType.UNKNOWN)
resource_ids = screen_state.get("resource_ids", set())
# 1. Action Compatibility Matrix (Whitelist Check)
# If the intent matches a known interaction type, check if the screen allows it.
for interaction_type, valid_screens in self.VALID_SCREENS.items():
if interaction_type in intent_lower:
if screen_type not in valid_screens:
logger.warning(
f"🛡️ [ContextGate] Blocked '{intent}' on {screen_type.name}: "
f"Not in valid screens {[s.name for s in valid_screens]}."
)
return False
break # Found the interaction type, don't check others
# ── Layer 0: Categorical Ban Matrix (instant, no dependencies) ──
matched_keyword = self._extract_interaction_keyword(intent_lower)
if matched_keyword is not None and screen_type != ScreenType.UNKNOWN:
allowed_screens = ALLOWED_SCREENS[matched_keyword]
if screen_type not in allowed_screens:
logger.debug(
f"🛡️ [ContextGate] BLOCKED '{intent}' on {screen_type.name} "
f"structurally impossible (allowed: {[s.name for s in allowed_screens]})"
)
return False
# 2. Structural Marker Verification
# Even on a valid screen, the required UI elements must be present.
for required_intent, markers in self.REQUIRED_MARKERS.items():
if required_intent in intent_lower:
has_marker = False
for marker in markers:
for rid in resource_ids:
if marker in rid.lower():
has_marker = True
break
if has_marker:
break
if not has_marker:
logger.warning(
f"🛡️ [ContextGate] Blocked '{intent}' on {screen_type.name}: "
f"No structural markers found {markers}."
)
return False
# ── Layer 1: Qdrant Learned Failures ──
if (
matched_keyword is not None
and screen_type != ScreenType.UNKNOWN
and self.context_memory is not None
and getattr(self.context_memory, "is_connected", False)
):
if not self.context_memory.is_allowed(intent_lower, screen_type.name):
logger.debug(
f"🛡️ [ContextGate] BLOCKED '{intent}' on {screen_type.name}" f"learned failure from Qdrant"
)
return False
# ── Default: Allow (Exploration) ──
return True
def get_valid_screens(self, intent: str) -> Optional[FrozenSet[ScreenType]]:
"""
Returns the set of screens where an interaction intent is structurally valid.
Used by the Planner for auto-routing when the goal can't be achieved on
the current screen.
Returns:
FrozenSet[ScreenType] if the intent maps to a known interaction, else None.
"""
keyword = self._extract_interaction_keyword(intent.lower())
if keyword is not None:
return ALLOWED_SCREENS[keyword]
return None
@staticmethod
def _extract_interaction_keyword(intent_lower: str) -> Optional[str]:
"""
Extracts the primary interaction keyword from an intent string.
Returns None if no interaction keyword is found (i.e., this is a navigation intent).
Uses word-boundary matching to prevent false positives:
- "follow" matches "follow user" but NOT "followers" or "following list"
- "like" matches "like post" but NOT "likelihood"
"""
import re
for kw in INTERACTION_KEYWORDS:
if re.search(rf"\b{kw}\b", intent_lower):
return kw
return None

View File

@@ -211,7 +211,7 @@ class IntentResolver:
rid = (node.resource_id or "").lower()
text = (node.text or "").lower()
if "composer_edittext" in rid or "message…" in text or "message..." in text:
logger.info(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
return node
if "last received message text" in intent_lower or "received message" in intent_lower:
@@ -220,7 +220,7 @@ class IntentResolver:
if msg_nodes:
# The last one in the XML is typically the most recent message at the bottom of the screen
latest_msg = msg_nodes[-1]
logger.info(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
logger.debug(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
return latest_msg
if "send message button" in intent_lower:
@@ -229,24 +229,30 @@ class IntentResolver:
desc = (node.content_desc or "").lower()
text = (node.text or "").lower()
if "send" in rid or "composer_button" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
logger.debug(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
return node
if "post author username" in intent_lower or "tap post username" in intent_lower:
for node in candidates:
if "row_feed_photo_profile_imageview" in (node.resource_id or "").lower():
logger.info(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
rid = (node.resource_id or "").lower()
if (
"row_feed_photo_profile_imageview" in rid
or "clips_author_profile_pic" in rid
or "reel_viewer_profile_picture" in rid
):
logger.debug(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
return node
for node in candidates:
if "row_feed_photo_profile_name" in (node.resource_id or "").lower():
logger.info(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
rid = (node.resource_id or "").lower()
if "row_feed_photo_profile_name" in rid or "clips_author_username" in rid or "reel_viewer_title" in rid:
logger.debug(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
return node
if "feed post content" in intent_lower or "post media content" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
if "row_feed_photo_imageview" in rid or "zoomable_view_container" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
return node
if "comment" in intent_lower and "button" in intent_lower:
@@ -266,14 +272,14 @@ class IntentResolver:
"row_feed_button_comment" in (node.resource_id or "").lower()
or "row_feed_textview_comments" in (node.resource_id or "").lower()
):
logger.info(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
logger.debug(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
return node
if "like" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
for node in candidates:
rid = (node.resource_id or "").lower()
if "row_feed_button_like" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found like button: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found like button: {rid}")
return node
if ("send" in intent_lower or "share" in intent_lower) and "post" in intent_lower and "button" in intent_lower:
@@ -281,7 +287,7 @@ class IntentResolver:
rid = (node.resource_id or "").lower()
desc = (node.content_desc or "").lower()
if "row_feed_button_share" in rid or "send post" in desc:
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
logger.debug(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
return node
if "add to story" in intent_lower:
@@ -293,28 +299,51 @@ class IntentResolver:
for node in candidates:
rid = (node.resource_id or "").lower()
if "row_feed_button_share" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found share button: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found share button: {rid}")
return node
if "save" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
for node in candidates:
rid = (node.resource_id or "").lower()
if "row_feed_button_save" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found save button: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found save button: {rid}")
return node
if "follow" in intent_lower and "button" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
if "profile_header_follow_button" in rid or "inline_follow_button" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
if (
"profile_header_follow_button" in rid
or "inline_follow_button" in rid
or "follow_list_row_large_follow_button" in rid
or "row_search_user_follow_button" in rid
or "profile_header_user_action_follow_button" in rid
):
logger.debug(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
return node
if "first post" in intent_lower or "first item" in intent_lower or "first search result" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
if "grid_card_layout_container" in rid or "image_button" in rid or "row_search_user" in rid:
logger.info(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
return node
if "heart" in intent_lower and "notification" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
desc = (node.content_desc or "").lower()
# Could be in top bar or bottom bar depending on IG version
if "notification" in rid or "newsfeed" in rid or "activity" in desc or "notification" in desc:
logger.debug(f"🎯 [Structural Fast-Path] Found notifications/heart icon: {rid or desc}")
return node
if ("inbox" in intent_lower or "direct message" in intent_lower) and "icon" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
desc = (node.content_desc or "").lower()
if "direct_tab" in rid or "inbox_button" in rid or "message" in desc:
logger.debug(f"🎯 [Structural Fast-Path] Found DM/inbox icon: {rid or desc}")
return node
if "story ring" in intent_lower or "story tray" in intent_lower:
@@ -354,11 +383,17 @@ class IntentResolver:
return story_nodes[0]
# --- Structural Grid Fast-Paths ---
if "first image post in profile grid" in intent_lower or "first post" in intent_lower:
if (
"first image post in profile grid" in intent_lower
or "first post" in intent_lower
or "first image" in intent_lower
):
for node in candidates:
desc = (node.content_desc or "").lower()
if "row 1, column 1" in desc:
logger.info(f"🎯 [Structural Fast-Path] Found first grid post: {node.resource_id} (desc: '{desc}')")
if "row 1" in desc and "column 1" in desc:
logger.debug(
f"🎯 [Structural Fast-Path] Found first grid post: {node.resource_id} (desc: '{desc}')"
)
return node
# --- Navigation Tab Fast-Paths ---
@@ -380,7 +415,7 @@ class IntentResolver:
for node in candidates:
rid = (node.resource_id or "").lower()
if rid.endswith(f":id/{resource_suffix}"):
logger.info(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
logger.debug(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
return node
# Priority 2: Fail Fast
@@ -418,7 +453,7 @@ class IntentResolver:
if semantic_candidates:
if len(semantic_candidates) == 1:
logger.info(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
logger.debug(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
return semantic_candidates[0]
else:
logger.info(
@@ -436,11 +471,9 @@ class IntentResolver:
if device is not None and (
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
):
print(f"DEBUG_INTENT: Entering Visual Discovery for '{intent_description}'")
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
return self._visual_discovery(intent_description, candidates, device, screen_height=screen_height)
print(f"DEBUG_INTENT: Falling back to Text-based VLM for '{intent_description}'")
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
# we enforce a strict failure.
@@ -659,6 +692,35 @@ class IntentResolver:
filtered_candidates.append(node)
candidates = filtered_candidates
# --- Reply Guard ---
# Prevents VLM from clicking the "Reply to story" or "Send message" input field
# when looking for general navigation or "next" buttons.
if (
"reply" not in intent_lower
and "message" not in intent_lower
and "comment" not in intent_lower
and "type" not in intent_lower
and "write" not in intent_lower
):
filtered_candidates = []
for node in candidates:
res_id = (node.resource_id or "").lower()
text = (node.text or "").lower()
cls_name = (node.class_name or "").lower()
if (
"reply" in res_id
or "message" in res_id
or "comment" in res_id
or "antworten" in text
or "send message" in text
or "nachricht" in text
or "edittext" in cls_name
):
logger.debug(f"🛡️ [Reply Guard] Filtered out input/message box: '{node.text}' ({node.resource_id})")
else:
filtered_candidates.append(node)
candidates = filtered_candidates
try:
annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates)
except Exception as e:
@@ -742,7 +804,6 @@ class IntentResolver:
use_local_edge=True,
images_b64=[annotated_b64],
)
print(f"DEBUG_INTENT: VLM RAW RESPONSE for '{intent_description}': {res}")
data = json.loads(res)
box_idx = self._parse_box_index(data)
selected = self._validate_and_get_node(box_idx, box_map)
@@ -871,7 +932,6 @@ class IntentResolver:
user_prompt=prompt,
use_local_edge=True,
)
print(f"DEBUG_INTENT: TEXT LLM RAW RESPONSE for '{intent_description}': {res}")
data = json.loads(res)
idx = data.get("selected_index")
if idx is not None and 0 <= idx < len(filtered_candidates):

View File

@@ -156,6 +156,7 @@ class ScreenIdentity:
"selected_tab": selected_tab,
"context": context,
"signature": signature,
"resource_ids": resource_ids,
}
def _classify_screen(
@@ -177,12 +178,14 @@ class ScreenIdentity:
# Priority 1: Content-creation overlays that block ALL navigation.
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
# EXCEPTION: If Qdrant has explicitly learned this screen as NORMAL (via LLM unlearning),
# we skip the structural check to prevent false-positive infinite loops.
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
browser_markers = ("ig_browser_text_title", "ig_browser_close_button", "ig_chrome_subsection")
if any(marker in ids_str for marker in creation_flow_markers):
if not is_normal_override and any(marker in ids_str for marker in creation_flow_markers):
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
return ScreenType.MODAL
if any(marker in ids_str for marker in browser_markers):
if not is_normal_override and any(marker in ids_str for marker in browser_markers):
logger.info("🛡️ [ScreenIdentity] In-App Browser detected → MODAL")
return ScreenType.MODAL
@@ -199,8 +202,20 @@ class ScreenIdentity:
"profile_header_business_category",
)
if any(marker in ids for marker in PROFILE_MARKERS):
# OWN_PROFILE is confirmed by the bottom tab OR the presence of 'edit' markers
if selected_tab == "profile_tab" or "profile_header_edit_profile_button" in ids:
# OWN_PROFILE detection priority cascade:
# 1. Selected tab == profile_tab (most reliable — structural)
# 2. Edit profile button present (structural)
# 3. Bot username found in visible text (semantic fallback)
is_own = False
if selected_tab == "profile_tab":
is_own = True
elif "profile_header_edit_profile_button" in ids:
is_own = True
elif text_lower:
if self.bot_username and self.bot_username in text_lower:
is_own = True
if is_own:
return ScreenType.OWN_PROFILE
return ScreenType.OTHER_PROFILE

View File

@@ -29,7 +29,10 @@ class QdrantBase:
try:
qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344")
self.client = QdrantClient(url=qdrant_url, timeout=10.0)
if qdrant_url == ":memory:":
self.client = QdrantClient(location=":memory:")
else:
self.client = QdrantClient(url=qdrant_url, timeout=10.0)
if self.client:
if self.client.collection_exists(collection_name):
@@ -208,7 +211,12 @@ class QdrantBase:
try:
res = self.client.retrieve(collection_name=self.collection_name, ids=[point_id])
if res:
self.client.delete(collection_name=self.collection_name, points_selector=[point_id])
from qdrant_client.models import PointIdsList
self.client.delete(
collection_name=self.collection_name,
points_selector=PointIdsList(points=[point_id]),
)
logger.info(
f"🗑️ [Qdrant] Purged poisoned memory vector from {self.collection_name} (UUID: {point_id[:8]}...)",
extra={"color": "\x1b[31m"},
@@ -1368,6 +1376,156 @@ class ParasocialCRMDB(QdrantBase):
return "\n".join(context_parts)
class FailureJournalDB(QdrantBase):
"""
P1-1: Crash Black Box.
Stores exact state context and intents that led to softlocks, crashes, or "LIE DETECTED"
phantom execution. Allows the system to learn what NOT to do.
"""
def __init__(self):
super().__init__(collection_name="gramaddict_failure_journal_v1")
def record_failure(self, screen_state: str, intent: str, error_msg: str):
if not self.is_connected:
return
failure_key = f"{screen_state}_{intent}"
vector = self._get_embedding(failure_key)
if not vector:
return
payload = {
"screen_state": screen_state,
"intent": intent,
"error_msg": error_msg,
"timestamp": time.time(),
}
self.upsert_point(
seed_string=failure_key,
vector=vector,
payload=payload,
log_success=f"📓 [FailureJournal] Recorded critical failure: {intent} on {screen_state}",
)
def is_known_failure(self, screen_state: str, intent: str, threshold: float = 0.95) -> bool:
if not self.is_connected:
return False
failure_key = f"{screen_state}_{intent}"
vector = self._get_embedding(failure_key)
if not vector:
return False
try:
results = self.client.query_points(
collection_name=self.collection_name,
query=vector,
limit=1,
).points
if results and results[0].score >= threshold:
logger.warning(
f"📓 [FailureJournal] Circuit Breaker: Preventing known fatal action '{intent}' on '{screen_state}'"
)
return True
except Exception:
pass
return False
class ContextMemoryDB(QdrantBase):
"""
Learns which intents are possible on which screens.
Replaces ContextGate's hardcoded VALID_SCREENS whitelist.
"""
def __init__(self):
super().__init__(collection_name="gramaddict_context_memory_v1", vector_size=128)
def _get_key(self, intent: str, screen_type: str) -> str:
return f"{intent}_{screen_type}"
def update_confidence(self, intent: str, screen_type: str, delta: float):
if not self.is_connected:
return
key = self._get_key(intent, screen_type)
try:
# Generate a consistent ID
point_id = self.generate_uuid(key)
points = self.client.retrieve(
collection_name=self.collection_name, ids=[point_id], with_payload=True, with_vectors=False
)
# Default neutral confidence for new state-intent pairs is 0.5
current_conf = 0.5
if points:
current_conf = points[0].payload.get("confidence", 0.5)
new_conf = max(0.0, min(1.0, current_conf + delta))
# Using zero-vector for fast KV-like lookup, since we rely entirely on exact point_id match
vector = [0.0] * self._vector_size
from qdrant_client.models import PointStruct
self.client.upsert(
collection_name=self.collection_name,
points=[
PointStruct(
id=point_id,
vector=vector,
payload={
"intent": intent,
"screen_type": screen_type,
"confidence": new_conf,
"updated_at": time.time(),
},
)
],
wait=True,
)
color = "\x1b[32m" if delta > 0 else "\x1b[31m"
symbol = "📈" if delta > 0 else "📉"
logger.info(
f"{symbol} [ContextMemory] Confidence for '{intent}' on '{screen_type}' adjusted to {new_conf:.2f} (delta: {delta:+.2f})",
extra={"color": color},
)
except Exception as e:
logger.debug(f"ContextMemory error: {e}")
def is_allowed(self, intent: str, screen_type: str) -> bool:
"""
Exploration vs Exploitation:
If we don't know (no entry or neutral confidence), allow it!
Only block if we have learned it fails consistently (< 0.2).
"""
if not self.is_connected:
return True # Fail-open for exploration
key = self._get_key(intent, screen_type)
try:
point_id = self.generate_uuid(key)
points = self.client.retrieve(
collection_name=self.collection_name, ids=[point_id], with_payload=True, with_vectors=False
)
if points:
conf = points[0].payload.get("confidence", 0.5)
if conf < 0.2:
logger.warning(
f"🛡️ [ContextMemory] Circuit Breaker: Blocked '{intent}' on '{screen_type}' "
f"(learned confidence {conf:.2f} < 0.2)"
)
return False
except Exception:
pass
return True # Allow exploration
def wipe_all_ai_caches():
"""
Wipes ALL global (non-user-specific) Qdrant AI caches.

View File

@@ -29,6 +29,7 @@ class SituationType(Enum):
OBSTACLE_MODAL = "obstacle_modal"
OBSTACLE_FOREIGN_APP = "obstacle_foreign_app"
OBSTACLE_SYSTEM = "obstacle_system"
OBSTACLE_KEYBOARD = "obstacle_keyboard"
DANGER_ACTION_BLOCKED = "danger_action_blocked"
@@ -337,6 +338,18 @@ class SituationalAwarenessEngine:
logger.info("📱 [SAE Perceive] System permission dialog explicitly detected.")
return SituationType.OBSTACLE_SYSTEM
# ── Keyboard Detection (Fast Path) ──
keyboard_pkgs = {
"com.google.android.inputmethod.latin",
"com.samsung.android.honeyboard",
"com.sec.android.inputmethod",
"com.touchtype.swiftkey",
"com.apple.android.inputmethod",
}
if any(pkg in keyboard_pkgs for pkg in packages):
logger.info("📱 [SAE Perceive] On-screen Keyboard explicitly detected. Treating as obstacle.")
return SituationType.OBSTACLE_KEYBOARD
# ── Foreign Environment Detection (package-based) ──
# If the main app package is completely absent from the UI hierarchy,
# OR if there's a dominant foreign package and no app package, we might have lost the app.

View File

@@ -118,7 +118,11 @@ class TelepathicEngine:
# 4. Track action
if track:
self._memory.track_click(intent_description, best_node, xml_string)
from GramAddict.core.perception.screen_identity import ScreenIdentity
screen_state = ScreenIdentity("").identify(xml_string)
screen_type = screen_state.get("screen_type").name if screen_state.get("screen_type") else "UNKNOWN"
self._memory.track_click(intent_description, best_node, xml_string, screen_type=screen_type)
# Translate to old GramAddict dict format for backward compatibility
return self._translate_node(best_node)