fix: VLM guard rejects following-count as 'hallucinated nav tab' + back-press circuit breaker
Root cause: VLM Structural Guard enforced 'must be at bottom' for ALL
NAV_INTENT_KEYWORDS including 'following'/'follower'. But these are
profile stats at Y≈246 (top of screen), not nav tabs. The inner
_structural_sanity_check correctly checks for 'tab' in intent before
enforcing bottom-zone requirement — the VLM guard was inconsistent.
Fix 1: Align VLM guard with inner guard — only enforce bottom-zone
requirement for intents explicitly containing 'tab'.
Fix 2: Add back-press circuit breaker (MAX_CONSECUTIVE_BACK=3). If GOAP
presses back 3 times on the same screen without any transition,
abort immediately to prevent exiting Instagram entirely.
95 unit tests pass.
This commit is contained in:
@@ -965,6 +965,8 @@ class GoalExecutor:
|
||||
steps_taken = []
|
||||
last_action = None
|
||||
last_screen_type = None
|
||||
consecutive_back_presses = 0
|
||||
MAX_CONSECUTIVE_BACK = 3
|
||||
explored_nav_actions = set()
|
||||
for step_num in range(max_steps):
|
||||
# PERCEIVE
|
||||
@@ -976,6 +978,7 @@ class GoalExecutor:
|
||||
f"📍 [GOAP State] Screen transitioned from {last_screen_type.name} to {screen_type.name}. Clearing explored actions."
|
||||
)
|
||||
explored_nav_actions.clear()
|
||||
consecutive_back_presses = 0 # Progress was made
|
||||
|
||||
# ── Loop Prevention: Mask Failed Actions ──
|
||||
MAX_RETRIES = 2
|
||||
@@ -1044,6 +1047,19 @@ class GoalExecutor:
|
||||
explored_nav_actions.add(action)
|
||||
# Reset failures for this action since it eventually succeeded
|
||||
self.action_failures[action] = 0
|
||||
|
||||
# ── Back-Press Circuit Breaker ──
|
||||
if action == "press back":
|
||||
consecutive_back_presses += 1
|
||||
if consecutive_back_presses >= MAX_CONSECUTIVE_BACK:
|
||||
logger.error(
|
||||
f"🛑 [GOAP] Back-pressed {MAX_CONSECUTIVE_BACK} times with no screen transition. "
|
||||
f"Aborting goal '{goal}' to prevent app exit."
|
||||
)
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
|
||||
return False
|
||||
else:
|
||||
consecutive_back_presses = 0
|
||||
else:
|
||||
self.action_failures[action] = self.action_failures.get(action, 0) + 1
|
||||
# Track failed actions in explored_nav_actions so the planner
|
||||
|
||||
@@ -2085,11 +2085,10 @@ class TelepathicEngine:
|
||||
# NAVIGATION TAB ENFORCEMENT:
|
||||
# Real navigation tabs (Home, Search, Reels, Store, Profile) are ALWAYS in the bottom zone.
|
||||
# If the bot is looking for a tab, forbid results that are too high up (likely hallucinations on comments/feed).
|
||||
if is_nav_intent:
|
||||
# Navigation tabs (Home, Search, Reels, News, Profile) are strictly at the bottom.
|
||||
# On many devices with virtual buttons, they are roughly at 0.94 - 0.98.
|
||||
# On full-screen gesture devices, they might be slightly higher.
|
||||
# We use 0.92 as a firm structural boundary.
|
||||
# IMPORTANT: Only enforce for intents explicitly containing "tab".
|
||||
# "following"/"follower" are profile stats at the TOP of the screen, not tabs.
|
||||
# This aligns with _structural_sanity_check which has the same "tab" guard.
|
||||
if is_nav_intent and "tab" in intent.lower():
|
||||
if match.get("y", 0) < screen_height * 0.92:
|
||||
logger.warning(
|
||||
f"🛡️ [Structural Guard] VLM hallucinated a navigation tab '{intent}' "
|
||||
|
||||
113
tests/unit/test_following_nav_guard.py
Normal file
113
tests/unit/test_following_nav_guard.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
🔴 RED TDD: VLM Structural Guard Inconsistency + GOAP Back-Press Loop
|
||||
|
||||
Bug A: VLM guard enforces "must be at bottom" for ALL nav intent keywords,
|
||||
but "following"/"follower" are PROFILE STATS, not nav tabs. The inner
|
||||
_structural_sanity_check correctly only enforces for "tab" intents.
|
||||
|
||||
Bug B: GOAP back-press loop has no circuit breaker. If the planner keeps
|
||||
pressing back on the same screen, it eventually exits Instagram.
|
||||
"""
|
||||
import pytest
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
from GramAddict.core.goap import GoalExecutor, ScreenType
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestVlmGuardFollowingConsistency:
|
||||
"""The VLM guard must NOT reject 'following' count elements at the top of the screen."""
|
||||
|
||||
def test_following_intent_allows_top_screen_elements(self):
|
||||
"""
|
||||
The 'following' count on a profile is at Y≈246 (top 10%).
|
||||
The VLM Structural Guard must NOT reject this as a 'hallucinated nav tab'.
|
||||
It's a profile stat, not a tab.
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2424
|
||||
|
||||
# Simulate the exact node the VLM found
|
||||
following_node = {
|
||||
"semantic_string": "text: '2,285 following', id context: 'row profile header following container'",
|
||||
"y": 246,
|
||||
"area": 5000,
|
||||
"class_name": "android.widget.TextView",
|
||||
"resource_id": "row_profile_header_following_container",
|
||||
}
|
||||
|
||||
intent = "tap following list"
|
||||
|
||||
# Inner structural guard should accept this (it has the "tab" check)
|
||||
is_valid = engine._structural_sanity_check(following_node, intent, screen_height)
|
||||
assert is_valid is True, (
|
||||
"Inner structural guard rejected 'following' element at Y=246. "
|
||||
"The intent 'tap following list' does not contain 'tab', so the nav-tab enforcement should not apply."
|
||||
)
|
||||
|
||||
def test_follower_intent_allows_top_screen_elements(self):
|
||||
"""Same bug for 'follower' count on a profile."""
|
||||
engine = TelepathicEngine()
|
||||
screen_height = 2424
|
||||
|
||||
follower_node = {
|
||||
"semantic_string": "text: '2,622 followers', id context: 'row profile header followers container'",
|
||||
"y": 246,
|
||||
"area": 5000,
|
||||
"class_name": "android.widget.TextView",
|
||||
"resource_id": "row_profile_header_followers_container",
|
||||
}
|
||||
|
||||
intent = "open followers list"
|
||||
|
||||
is_valid = engine._structural_sanity_check(follower_node, intent, screen_height)
|
||||
assert is_valid is True, (
|
||||
"Inner structural guard rejected 'followers' element at Y=246."
|
||||
)
|
||||
|
||||
|
||||
class TestGoapBackPressCircuitBreaker:
|
||||
"""GOAP must detect and abort back-press loops on the same screen."""
|
||||
|
||||
def test_consecutive_back_presses_on_same_screen_aborts(self):
|
||||
"""
|
||||
If the GOAP planner presses back 3+ times on the same screen type
|
||||
without any screen transition, it should abort instead of continuing
|
||||
to press back until it exits the app.
|
||||
"""
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device._get_current_app.return_value = "com.instagram.android"
|
||||
|
||||
goap = GoalExecutor(device, bot_username="testbot")
|
||||
goap._sae = MagicMock()
|
||||
goap._sae.ensure_clear_screen.return_value = True
|
||||
|
||||
# Simulate being stuck on HOME_FEED with only back available
|
||||
call_count = 0
|
||||
def mock_perceive():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["press back"],
|
||||
"context": {},
|
||||
"selected_tab": "feed_tab",
|
||||
}
|
||||
|
||||
goap.perceive = mock_perceive
|
||||
|
||||
# Mock _execute_action to always succeed (pressing back "works" but stays on same screen)
|
||||
goap._execute_action = MagicMock(return_value=True)
|
||||
|
||||
result = goap.achieve("open following list", max_steps=15)
|
||||
|
||||
# The bot should NOT have pressed back more than 3 times
|
||||
back_calls = [
|
||||
c for c in goap._execute_action.call_args_list
|
||||
if c[0][0] == "press back"
|
||||
]
|
||||
assert len(back_calls) <= 3, (
|
||||
f"GOAP pressed back {len(back_calls)} times on the same screen. "
|
||||
"It should abort after 3 consecutive back-presses with no progress."
|
||||
)
|
||||
assert result is False, "GOAP should fail when stuck in a back-press loop."
|
||||
Reference in New Issue
Block a user