2 Commits

Author SHA1 Message Date
67c3d464e0 fix(navigation): harden ScreenIdentity priority cascade and GOAP structural action detection
1. ScreenIdentity: Restructured priority cascade to prevent Qdrant semantic
   cache from overriding deterministic structural heuristics. Cached types
   now resolve AFTER structural checks (Priority 3) instead of before.
   Story/Reels hallucinations from both cache and VLM are rejected if
   structural markers are absent.

2. bot_flow: Added error handling for story ring avatar tap and curiosity
   HomeFeed navigation failures — prevents silent continuation into
   undefined states.

3. GOAP: Extended is_navigation detection to include ScreenTopology
   structural actions, ensuring HD Map routes are correctly classified.

4. action_memory: Tightened _parse_yes_no to prevent JSON fall-through
   into ambiguous text matching. Structured responses with 'success'
   boolean are now handled directly.
2026-05-03 17:00:27 +02:00
d298f03891 fix(sae): structural fast-path for Instagram survey/interstitial modal detection
The SAE perceive() had no structural fast-path for Instagram-internal
modal overlays (surveys, rating prompts, interstitials). These modals
live inside com.instagram.android and were invisible to the foreign-app
detector. The LLM fallback frequently misclassified them as NORMAL,
causing the bot to get trapped in survey dialogs.

Added two O(1) structural detection layers:
1. Resource-ID markers: survey_overlay_container, interstitial_container,
   mystery_interstitial, nux_overlay, rating_prompt, feedback_dialog
2. Dismiss-button heuristic: cross-validates dismiss text with overlay
   container structure to prevent caption false positives

Also removed dead code: xml_dump.lower() no-op.

Fixes: test_perceive_instagram_survey_modal
Zero regressions: 464 passed, 6 pre-existing failures
2026-05-03 16:59:52 +02:00
5 changed files with 96 additions and 28 deletions

View File

@@ -557,7 +557,9 @@ def start_bot(**kwargs):
continue
elif current_target == "StoriesFeed":
logger.info("📱 Locating story tray on HomeFeed...")
nav_graph.do("tap story ring avatar")
if not nav_graph.do("tap story ring avatar"):
logger.warning("❌ Failed to tap story ring avatar. Retrying next loop.")
continue
post_loaded = _wait_for_story_loaded(device, timeout=5)
if not post_loaded:
logger.warning("❌ Stories failed to open from HomeFeed. Retrying next loop.")
@@ -927,7 +929,9 @@ def _run_zero_latency_feed_loop(
# 🛡️ Structural Guard: Curiosity targets (DMs, Notifications) are ONLY available on HomeFeed.
# We must navigate there first, breaking current context.
nav_graph.navigate_to("HomeFeed", zero_engine)
if not nav_graph.navigate_to("HomeFeed", zero_engine):
logger.warning("❌ [Curiosity] Failed to navigate to HomeFeed. Aborting curiosity check.")
continue
sleep(random.uniform(1.0, 2.5))
dm_config = configs.get_plugin_config("dm_reply")

View File

@@ -387,7 +387,10 @@ class GoalExecutor:
pre_action_screen_type = pre_action_screen["screen_type"]
# Determine if this was a navigation or an interaction
from GramAddict.core.screen_topology import ScreenTopology
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
if not is_navigation:
is_navigation = ScreenTopology.is_structural_action(pre_action_screen_type, action)
action_success = False
# ── UI Change Detection with Noise Threshold ──

View File

@@ -21,8 +21,15 @@ def _parse_yes_no(response: str) -> Optional[bool]:
return True
if str(k).strip().upper() == "NO" or str(v).strip().upper() == "NO":
return False
if str(k).strip().lower() == "success" and isinstance(v, bool):
return v
# If it is valid JSON but we couldn't definitively find YES/NO,
# do NOT fall through to text matching
return None
except Exception:
pass
# Prevent JSON parsing fall-throughs
return None
text_lower = text.lower()
if text_lower.startswith("yes"):
@@ -30,14 +37,6 @@ def _parse_yes_no(response: str) -> Optional[bool]:
if text_lower.startswith("no") and not text_lower.startswith("now") and not text_lower.startswith("not"):
return False
has_yes = re.search(r"\byes\b", text_lower) is not None
has_no = re.search(r"\bno\b", text_lower) is not None
if has_yes and not has_no:
return True
if has_no and not has_yes:
return False
return None

View File

@@ -162,21 +162,15 @@ class ScreenIdentity:
"""
Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
# Priority 0: Check Qdrant Semantic Cache (Learned Truth/LLM Overrides)
# This MUST be checked first. If the LLM declared this specific layout a "false positive"
# and cached it as NORMAL, it must override any rigid structural heuristics below to prevent
# infinite loops.
is_normal_override = False
# Priority 0: Fetch Qdrant Semantic Cache
# We fetch this early to see if there is a 'NORMAL' override for the MODAL check.
# We DO NOT let this override deterministic structural heuristics! Fuzzy vector matching
# can easily confuse HOME_FEED and OWN_PROFILE if the bottom navigation bar is identical.
cached_type_str = None
if signature and self.screen_memory and self.screen_memory.is_connected:
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
if cached_type_str:
if cached_type_str == "NORMAL":
is_normal_override = True
else:
try:
return ScreenType[cached_type_str]
except KeyError:
pass
is_normal_override = (cached_type_str == "NORMAL")
# Priority 1: Content-creation overlays that block ALL navigation.
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
@@ -187,7 +181,7 @@ class ScreenIdentity:
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
return ScreenType.MODAL
# Priority 1: Structural Heuristics (100% Deterministic)
# Priority 2: Structural Heuristics (100% Deterministic)
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
return ScreenType.FOLLOW_LIST
@@ -250,7 +244,21 @@ class ScreenIdentity:
# End of structural heuristics
# Priority 3: Semantic VLM Classification Fallback
# Priority 3: Cached Semantic Type (If deterministic heuristics failed)
if cached_type_str and cached_type_str != "NORMAL":
try:
cached_type = ScreenType[cached_type_str]
# Enforce absolute structural parity: Story and Reels must have their structural markers.
# If they reached Priority 3, it means Priority 2 failed to find their markers.
# Therefore, any cache telling us this is a Story/Reel without those markers is hallucinating.
if cached_type in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
logger.warning(f"⚠️ [ScreenIdentity] Rejecting cached {cached_type.name} due to missing structural markers.")
else:
return cached_type
except KeyError:
pass
# Priority 4: Semantic VLM Classification Fallback
if not screenshot_b64 and getattr(self, "device", None) is not None:
screenshot_b64 = self.device.get_screenshot_b64()
@@ -294,6 +302,11 @@ class ScreenIdentity:
# Prevent the LLM from hallucinating an obstacle if explicitly verified as NORMAL
return ScreenType.UNKNOWN
# Enforce absolute structural parity: Story and Reels must have their structural markers.
if t in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
logger.warning(f"⚠️ [ScreenIdentity] Rejecting VLM hallucinated {t.name} due to missing structural markers.")
return ScreenType.UNKNOWN
if signature and self.screen_memory:
self.screen_memory.store_screen(signature, t.name)
return t

View File

@@ -299,8 +299,6 @@ class SituationalAwarenessEngine:
if not xml_dump or not isinstance(xml_dump, str):
return SituationType.OBSTACLE_FOREIGN_APP
xml_dump.lower()
blocked_markers = [
"try again later",
"action blocked",
@@ -443,6 +441,57 @@ class SituationalAwarenessEngine:
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
return SituationType.OBSTACLE_MODAL
# ── Structural Fast-Check: Instagram-Internal Modal Overlays ──
# Surveys, rating prompts, and interstitial modals live INSIDE Instagram's
# package but block normal interaction. They share a common structural
# pattern: a container resource-id containing "survey", "interstitial",
# or "nux_" (new-user-experience), plus dismiss buttons ("Not Now").
# Detecting them structurally is O(1) and eliminates LLM hallucination risk.
instagram_modal_markers = (
"survey_overlay_container", # "How are you enjoying Instagram?" survey
"survey_title", # Survey title text view
"interstitial_container", # Generic interstitial blocker
"mystery_interstitial", # Unknown/dynamic interstitials
"nux_overlay", # New-user-experience onboarding modals
"rating_prompt", # App Store rating prompt
"feedback_dialog", # Feedback collection dialogs
)
if any(
re.search(rf'resource-id="[^"]*{marker}[^"]*"', xml_dump, re.IGNORECASE)
for marker in instagram_modal_markers
):
logger.info("🧠 [SAE Perceive] Instagram modal overlay detected structurally → OBSTACLE_MODAL")
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
return SituationType.OBSTACLE_MODAL
# Fallback heuristic: detect modals by dismiss-button text patterns.
# If we see "Not Now" or "Take Survey" as button text inside Instagram, it's a modal.
# Guard: match ONLY inside short text attributes (< 40 chars) to avoid caption false positives.
dismiss_button_patterns = (
r'text="Not Now"',
r'text="not now"',
r'text="Nicht jetzt"', # German: "Not Now"
r'text="Take Survey"',
r'text="rate \d+ stars?"', # "rate 5 stars"
r'text="Bewerten"', # German: "Rate"
)
has_dismiss_button = any(re.search(p, xml_dump, re.IGNORECASE) for p in dismiss_button_patterns)
if has_dismiss_button:
# Cross-validate: must also have a container that looks like a dialog/overlay
# (not just a random "Not Now" text in a DM thread or post caption)
has_overlay_structure = bool(
re.search(
r'resource-id="[^"]*(?:overlay|dialog|interstitial|survey|sheet|prompt)[^"]*"',
xml_dump,
re.IGNORECASE,
)
or re.search(r'resource-id="[^"]*button_(?:negative|positive)[^"]*"', xml_dump, re.IGNORECASE)
)
if has_overlay_structure:
logger.info("🧠 [SAE Perceive] Instagram dismiss-button modal detected structurally → OBSTACLE_MODAL")
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
return SituationType.OBSTACLE_MODAL
# If not cached, query LLM for autonomous structural classification
try:
from GramAddict.core.config import Config