fix(navigation): harden ScreenIdentity priority cascade and GOAP structural action detection
1. ScreenIdentity: Restructured priority cascade to prevent Qdrant semantic cache from overriding deterministic structural heuristics. Cached types now resolve AFTER structural checks (Priority 3) instead of before. Story/Reels hallucinations from both cache and VLM are rejected if structural markers are absent. 2. bot_flow: Added error handling for story ring avatar tap and curiosity HomeFeed navigation failures — prevents silent continuation into undefined states. 3. GOAP: Extended is_navigation detection to include ScreenTopology structural actions, ensuring HD Map routes are correctly classified. 4. action_memory: Tightened _parse_yes_no to prevent JSON fall-through into ambiguous text matching. Structured responses with 'success' boolean are now handled directly.
This commit is contained in:
@@ -557,7 +557,9 @@ def start_bot(**kwargs):
|
||||
continue
|
||||
elif current_target == "StoriesFeed":
|
||||
logger.info("📱 Locating story tray on HomeFeed...")
|
||||
nav_graph.do("tap story ring avatar")
|
||||
if not nav_graph.do("tap story ring avatar"):
|
||||
logger.warning("❌ Failed to tap story ring avatar. Retrying next loop.")
|
||||
continue
|
||||
post_loaded = _wait_for_story_loaded(device, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning("❌ Stories failed to open from HomeFeed. Retrying next loop.")
|
||||
@@ -927,7 +929,9 @@ def _run_zero_latency_feed_loop(
|
||||
|
||||
# 🛡️ Structural Guard: Curiosity targets (DMs, Notifications) are ONLY available on HomeFeed.
|
||||
# We must navigate there first, breaking current context.
|
||||
nav_graph.navigate_to("HomeFeed", zero_engine)
|
||||
if not nav_graph.navigate_to("HomeFeed", zero_engine):
|
||||
logger.warning("❌ [Curiosity] Failed to navigate to HomeFeed. Aborting curiosity check.")
|
||||
continue
|
||||
sleep(random.uniform(1.0, 2.5))
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
|
||||
@@ -387,7 +387,10 @@ class GoalExecutor:
|
||||
pre_action_screen_type = pre_action_screen["screen_type"]
|
||||
|
||||
# Determine if this was a navigation or an interaction
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
|
||||
if not is_navigation:
|
||||
is_navigation = ScreenTopology.is_structural_action(pre_action_screen_type, action)
|
||||
action_success = False
|
||||
|
||||
# ── UI Change Detection with Noise Threshold ──
|
||||
|
||||
@@ -21,8 +21,15 @@ def _parse_yes_no(response: str) -> Optional[bool]:
|
||||
return True
|
||||
if str(k).strip().upper() == "NO" or str(v).strip().upper() == "NO":
|
||||
return False
|
||||
if str(k).strip().lower() == "success" and isinstance(v, bool):
|
||||
return v
|
||||
|
||||
# If it is valid JSON but we couldn't definitively find YES/NO,
|
||||
# do NOT fall through to text matching
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
# Prevent JSON parsing fall-throughs
|
||||
return None
|
||||
|
||||
text_lower = text.lower()
|
||||
if text_lower.startswith("yes"):
|
||||
@@ -30,14 +37,6 @@ def _parse_yes_no(response: str) -> Optional[bool]:
|
||||
if text_lower.startswith("no") and not text_lower.startswith("now") and not text_lower.startswith("not"):
|
||||
return False
|
||||
|
||||
has_yes = re.search(r"\byes\b", text_lower) is not None
|
||||
has_no = re.search(r"\bno\b", text_lower) is not None
|
||||
|
||||
if has_yes and not has_no:
|
||||
return True
|
||||
if has_no and not has_yes:
|
||||
return False
|
||||
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -162,21 +162,15 @@ class ScreenIdentity:
|
||||
"""
|
||||
Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
|
||||
# Priority 0: Check Qdrant Semantic Cache (Learned Truth/LLM Overrides)
|
||||
# This MUST be checked first. If the LLM declared this specific layout a "false positive"
|
||||
# and cached it as NORMAL, it must override any rigid structural heuristics below to prevent
|
||||
# infinite loops.
|
||||
is_normal_override = False
|
||||
# Priority 0: Fetch Qdrant Semantic Cache
|
||||
# We fetch this early to see if there is a 'NORMAL' override for the MODAL check.
|
||||
# We DO NOT let this override deterministic structural heuristics! Fuzzy vector matching
|
||||
# can easily confuse HOME_FEED and OWN_PROFILE if the bottom navigation bar is identical.
|
||||
cached_type_str = None
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
if cached_type_str == "NORMAL":
|
||||
is_normal_override = True
|
||||
else:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
is_normal_override = (cached_type_str == "NORMAL")
|
||||
|
||||
# Priority 1: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
@@ -187,7 +181,7 @@ class ScreenIdentity:
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 1: Structural Heuristics (100% Deterministic)
|
||||
# Priority 2: Structural Heuristics (100% Deterministic)
|
||||
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
|
||||
return ScreenType.FOLLOW_LIST
|
||||
|
||||
@@ -250,7 +244,21 @@ class ScreenIdentity:
|
||||
|
||||
# End of structural heuristics
|
||||
|
||||
# Priority 3: Semantic VLM Classification Fallback
|
||||
# Priority 3: Cached Semantic Type (If deterministic heuristics failed)
|
||||
if cached_type_str and cached_type_str != "NORMAL":
|
||||
try:
|
||||
cached_type = ScreenType[cached_type_str]
|
||||
# Enforce absolute structural parity: Story and Reels must have their structural markers.
|
||||
# If they reached Priority 3, it means Priority 2 failed to find their markers.
|
||||
# Therefore, any cache telling us this is a Story/Reel without those markers is hallucinating.
|
||||
if cached_type in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
|
||||
logger.warning(f"⚠️ [ScreenIdentity] Rejecting cached {cached_type.name} due to missing structural markers.")
|
||||
else:
|
||||
return cached_type
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Priority 4: Semantic VLM Classification Fallback
|
||||
if not screenshot_b64 and getattr(self, "device", None) is not None:
|
||||
screenshot_b64 = self.device.get_screenshot_b64()
|
||||
|
||||
@@ -294,6 +302,11 @@ class ScreenIdentity:
|
||||
# Prevent the LLM from hallucinating an obstacle if explicitly verified as NORMAL
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
# Enforce absolute structural parity: Story and Reels must have their structural markers.
|
||||
if t in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
|
||||
logger.warning(f"⚠️ [ScreenIdentity] Rejecting VLM hallucinated {t.name} due to missing structural markers.")
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
if signature and self.screen_memory:
|
||||
self.screen_memory.store_screen(signature, t.name)
|
||||
return t
|
||||
|
||||
Reference in New Issue
Block a user