mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: ScreenTopology HD Map + graph-aware GOAP routing + NAV keyword split

Browse Source 
FUNDAMENTAL ARCHITECTURE OVERHAUL:

1. ScreenTopology HD Map (NEW):
   Pure-data BFS pathfinding between Instagram screens.
   Zero runtime dependencies. The GOAP planner's GPS.
   Knows: HOME_FEED → tap profile tab → OWN_PROFILE → tap following list → FOLLOW_LIST

2. Graph-Aware GOAP Planning:
   GoalPlanner._plan_navigation() now consults ScreenTopology FIRST.
   From HOME_FEED, goal 'open following list' returns 'tap profile tab'
   (intermediate hop) instead of blind 'open following list' (impossible).
   Autonomous discovery and Qdrant knowledge kept as fallbacks.

3. NAV Keyword Split:
   NAV_INTENT_KEYWORDS → NAV_ZONE_BYPASS_KEYWORDS + NAV_TAB_KEYWORDS
   Ends the Structural Guard civil war where 'following' was both
   'allowed in nav zone' AND 'must be at bottom' simultaneously.

4. QNavGraph Deduplication:
   _find_path() delegates to ScreenTopology.find_route() (SSOT).
   core_nodes seed generated from ScreenTopology.TRANSITIONS.

115 unit tests pass. Zero regressions.
This commit is contained in:
Marc Mintel
2026-04-24 22:33:58 +02:00
parent 82bf931b0e
commit d69da4c974
7 changed files with 462 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
GramAddict/core/goap.py
View File

@@ -758,7 +758,14 @@ class GoalPlanner:
selected_tab: Optional[str] = None,
explored_nav_actions: set = None,
) -> Optional[str]:
"""If we're on the wrong screen, figure out how to navigate."""
"""If we're on the wrong screen, figure out how to navigate.
Strategy (priority order):
1. HD Map (ScreenTopology BFS) — deterministic, pre-computed routes
2. Learned Knowledge (Qdrant) — dynamic discovery from past sessions
3. Autonomous Discovery — linguistic matching + VLM intent
"""
from GramAddict.core.screen_topology import ScreenTopology
# 0. Aversive Filter: Remove known traps from available actions
safe_available = []
@@ -769,10 +776,30 @@ class GoalPlanner:
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
available = safe_available
# 1. Get required screens for this goal from knowledge
# ── 1. HD Map Routing (Primary Strategy) ──
target_screen = ScreenTopology.goal_to_target_screen(goal)
if target_screen and target_screen != screen_type:
route = ScreenTopology.find_route(screen_type, target_screen)
if route:
next_action, next_screen = route[0]
# Verify action isn't explored/trapped
if next_action not in (explored_nav_actions or set()):
if not self.knowledge.is_trap(screen_type, next_action):
route_desc = " → ".join(s.name for _, s in route)
logger.info(
f"🗺️ [HD Map] Route: {screen_type.name} → {route_desc}. "
f"Next action: '{next_action}'"
)
return next_action
else:
logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.")
else:
logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.")
# ── 2. Learned Knowledge (Qdrant) ──
required_screens = self.knowledge.get_requirements(goal)
# 2. Blank Start Discovery (if knowledge is empty)
# ── 3. Autonomous Discovery (Blank Start fallback) ──
if not required_screens:
logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.")
@@ -784,15 +811,12 @@ class GoalPlanner:
for action in available:
if any(w in action.lower() for w in goal_words):
# Ensure it doesn't lead to a completely contradictory known state (if we knew the target).
# But since we are in a blank start, any linguistic match is a highly probable path!
logger.info(
f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{goal}'"
)
return action
# We don't know the required screen or path. Let the TelepathicEngine figure out
# what button to press based on the pure goal text!
# Return raw intent for TelepathicEngine discovery
if explored_nav_actions and goal in explored_nav_actions:
logger.info(
f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking."
@@ -801,37 +825,31 @@ class GoalPlanner:
else:
return goal
# 3. If we're already on an acceptable screen, no navigation needed
# 4. If we're already on an acceptable screen, no navigation needed
if screen_type in required_screens:
return None
# 4. Find the action we need to take
# 5. Find the action we need to take (from learned knowledge)
for target_screen in required_screens:
known_action = self.knowledge.get_action_for_screen(target_screen)
if not known_action:
logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...")
# Check ALL available actions if one linguistically aligns with the target screen
# Or just let VLM figure it out by returning an intention.
screen_friendly_name = target_screen.name.replace("_", " ").lower()
# Semantic Heuristic Match on dynamically perceived actions
goal_words = [w.rstrip("s") for w in screen_friendly_name.split() if len(w) > 3]
for action in available:
if any(w in action.lower() for w in goal_words):
# Verify we don't know this leads somewhere else
known_target = self.knowledge.get_screen_for_action(action)
if known_target and known_target != target_screen:
continue # Trap prevention
continue
logger.info(
f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'"
)
return action
# If no perceived action matches, return intent for TelepathicEngine
return f"navigate to {screen_friendly_name}"
else:
if known_action in available:
@@ -840,7 +858,6 @@ class GoalPlanner:
# If no targeted navigation works, try going back first
if "press back" in available:
# Only press back if we aren't currently on the required screen (handled in step 3)
return "press back"
return None

79
GramAddict/core/q_nav_graph.py
View File

@@ -9,6 +9,7 @@ from GramAddict.core.compiler_engine import VLMCompilerEngine
from GramAddict.core.qdrant_memory import NavigationMemoryDB
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
from GramAddict.core.goap import GoalExecutor, ScreenType
from GramAddict.core.screen_topology import ScreenTopology
logger = logging.getLogger(__name__)
@@ -37,18 +38,34 @@ class QNavGraph:
def _load_graph(self):
"""Loads the topological map from Qdrant. Merges with core seeds to guarantee baseline navigation."""
"""Loads the topological map from Qdrant. Merges with core seeds from ScreenTopology (SSOT)."""
logger.debug("🌐 [NavGraph] Syncing topological map with Qdrant...")
self.nodes = self.nav_memory.get_all_transitions()
core_nodes = {
"HomeFeed": {"transitions": {"tap_explore_tab": "ExploreFeed", "tap_profile_tab": "OwnProfile", "tap_message_icon": "MessageInbox"}},
"ExploreFeed": {"transitions": {"tap_home_tab": "HomeFeed"}},
"OwnProfile": {"transitions": {"tap_home_tab": "HomeFeed", "tap_following_list": "FollowingList"}},
"MessageInbox": {"transitions": {"tap_back": "HomeFeed"}},
"FollowingList": {"transitions": {"tap_back": "OwnProfile"}},
"UNKNOWN": {"transitions": {"tap_home_tab": "HomeFeed"}}
}
# Generate core_nodes from ScreenTopology (single source of truth)
core_nodes = {}
for screen_type, transitions in ScreenTopology.TRANSITIONS.items():
# Map ScreenType enum names to QNavGraph string names
screen_name_map = {
ScreenType.HOME_FEED: "HomeFeed",
ScreenType.EXPLORE_GRID: "ExploreFeed",
ScreenType.REELS_FEED: "ReelsFeed",
ScreenType.OWN_PROFILE: "OwnProfile",
ScreenType.DM_INBOX: "MessageInbox",
ScreenType.FOLLOW_LIST: "FollowingList",
ScreenType.OTHER_PROFILE: "OtherProfile",
ScreenType.UNKNOWN: "UNKNOWN",
}
node_name = screen_name_map.get(screen_type)
if not node_name:
continue
node_transitions = {}
for action, target_screen in transitions.items():
# Convert action format: "tap profile tab" → "tap_profile_tab"
action_key = action.replace(" ", "_")
target_name = screen_name_map.get(target_screen, target_screen.name)
node_transitions[action_key] = target_name
core_nodes[node_name] = {"transitions": node_transitions}
# Merge core nodes into loaded nodes
for node, data in core_nodes.items():
@@ -135,25 +152,29 @@ class QNavGraph:
return self.goap._execute_action(goal)
def _find_path(self, start: str, end: str):
if start == end: return []
if start not in self.nodes: return None
queue = [(start, [])]
visited = set()
while queue:
current, path = queue.pop(0)
if current == end:
return path
visited.add(current)
transitions = self.nodes.get(current, {}).get("transitions", {})
for action, next_state in transitions.items():
if next_state not in visited:
queue.append((next_state, path + [action]))
return None
"""Delegates to ScreenTopology for BFS pathfinding (SSOT)."""
# Map string names to ScreenType for ScreenTopology
name_to_screen = {
"HomeFeed": ScreenType.HOME_FEED,
"ExploreFeed": ScreenType.EXPLORE_GRID,
"ReelsFeed": ScreenType.REELS_FEED,
"OwnProfile": ScreenType.OWN_PROFILE,
"MessageInbox": ScreenType.DM_INBOX,
"FollowingList": ScreenType.FOLLOW_LIST,
"OtherProfile": ScreenType.OTHER_PROFILE,
"UNKNOWN": ScreenType.UNKNOWN,
}
from_screen = name_to_screen.get(start)
to_screen = name_to_screen.get(end)
if not from_screen or not to_screen:
return None
route = ScreenTopology.find_route(from_screen, to_screen)
if route is None:
return None
# Convert back to QNavGraph action format: "tap profile tab" → "tap_profile_tab"
return [action.replace(" ", "_") for action, _ in route]
def _clear_anomaly_obstacles(self, max_attempts=2, xml_dump: str = None) -> bool:
"""

135
GramAddict/core/screen_topology.py Normal file
View File

@@ -0,0 +1,135 @@
"""
ScreenTopology — The Instagram HD Map
Pure-data BFS pathfinding between Instagram screen states.
Zero dependencies on device, VLM, Qdrant, or any runtime state.
This is the bot's GPS: it knows HOW to get from screen A to screen B
before the bot starts moving. The GOAP planner consults this map
as its primary routing strategy.
"""
from collections import deque
from enum import Enum
from typing import Dict, List, Optional, Tuple
from GramAddict.core.goap import ScreenType
class ScreenTopology:
"""
Topological HD Map of Instagram's screen graph.
Provides BFS pathfinding between any two ScreenTypes.
All transitions use the same action string format as
the TelepathicEngine intent system — no translation needed.
"""
# ── The Map: ScreenType → {action_string → ScreenType} ──
# These are structural facts about Instagram's UI, not learned behavior.
# They survive blank_start because they describe the app's architecture.
TRANSITIONS: Dict[ScreenType, Dict[str, ScreenType]] = {
ScreenType.HOME_FEED: {
"tap explore tab": ScreenType.EXPLORE_GRID,
"tap profile tab": ScreenType.OWN_PROFILE,
"tap reels tab": ScreenType.REELS_FEED,
"tap messages tab": ScreenType.DM_INBOX,
},
ScreenType.EXPLORE_GRID: {
"tap home tab": ScreenType.HOME_FEED,
"tap profile tab": ScreenType.OWN_PROFILE,
"tap reels tab": ScreenType.REELS_FEED,
},
ScreenType.REELS_FEED: {
"tap home tab": ScreenType.HOME_FEED,
"tap explore tab": ScreenType.EXPLORE_GRID,
"tap profile tab": ScreenType.OWN_PROFILE,
},
ScreenType.OWN_PROFILE: {
"tap home tab": ScreenType.HOME_FEED,
"tap explore tab": ScreenType.EXPLORE_GRID,
"tap reels tab": ScreenType.REELS_FEED,
"tap following list": ScreenType.FOLLOW_LIST,
},
ScreenType.DM_INBOX: {
"press back": ScreenType.HOME_FEED,
},
ScreenType.FOLLOW_LIST: {
"press back": ScreenType.OWN_PROFILE,
},
ScreenType.OTHER_PROFILE: {
"press back": ScreenType.HOME_FEED,
"tap home tab": ScreenType.HOME_FEED,
},
ScreenType.UNKNOWN: {
"tap home tab": ScreenType.HOME_FEED,
},
}
# ── Goal → ScreenType mapping ──
# Maps natural-language goals to their target screen.
_GOAL_MAP: Dict[str, ScreenType] = {
"open home feed": ScreenType.HOME_FEED,
"open home": ScreenType.HOME_FEED,
"open explore feed": ScreenType.EXPLORE_GRID,
"open explore": ScreenType.EXPLORE_GRID,
"open reels": ScreenType.REELS_FEED,
"open profile": ScreenType.OWN_PROFILE,
"learn own profile": ScreenType.OWN_PROFILE,
"open messages": ScreenType.DM_INBOX,
"open following list": ScreenType.FOLLOW_LIST,
"open followers list": ScreenType.FOLLOW_LIST,
}
@classmethod
def find_route(
cls, from_screen: ScreenType, to_screen: ScreenType
) -> Optional[List[Tuple[str, ScreenType]]]:
"""
BFS shortest path from from_screen to to_screen.
Returns:
[] if already there,
[(action, resulting_screen), ...] for a path,
None if unreachable.
"""
if from_screen == to_screen:
return []
queue: deque = deque()
queue.append((from_screen, []))
visited = {from_screen}
while queue:
current, path = queue.popleft()
transitions = cls.TRANSITIONS.get(current, {})
for action, next_screen in transitions.items():
if next_screen == to_screen:
return path + [(action, next_screen)]
if next_screen not in visited:
visited.add(next_screen)
queue.append((next_screen, path + [(action, next_screen)]))
return None # Unreachable
@classmethod
def get_transitions(cls, screen: ScreenType) -> Dict[str, ScreenType]:
"""Get all known transitions from a screen."""
return dict(cls.TRANSITIONS.get(screen, {}))
@classmethod
def goal_to_target_screen(cls, goal: str) -> Optional[ScreenType]:
"""Map a goal string to its target ScreenType. Returns None for non-navigation goals."""
goal_lower = goal.lower().strip()
# Exact match first
if goal_lower in cls._GOAL_MAP:
return cls._GOAL_MAP[goal_lower]
# Substring match for flexibility
for key, screen in cls._GOAL_MAP.items():
if key in goal_lower:
return screen
return None

50
GramAddict/core/telepathic_engine.py
View File

@@ -24,9 +24,17 @@ MAX_BUTTON_AREA = 150000 # Buttons/icons should be smaller than this (px²)
MAX_CONTAINER_AREA = 500000 # Anything above this is a full-screen container
# ── Nav Intent Keywords (SSOT — used by all structural guards) ──
# Any intent containing these keywords is classified as "navigation",
# allowing the structural guard to accept elements in the nav bar zone.
NAV_INTENT_KEYWORDS = [
#
# TWO purpose-specific sets prevent the Guard Civil War:
#
# NAV_ZONE_BYPASS_KEYWORDS: Intents ALLOWED to target elements in the
# bottom nav bar zone. Broad — includes tabs, DMs, profile stats.
#
# NAV_TAB_KEYWORDS: Intents that REQUIRE their target to be at the bottom.
# Narrow — only actual navigation tabs. "following"/"follower" are profile
# stats at the TOP of the screen, not tabs.
NAV_ZONE_BYPASS_KEYWORDS = [
"tab",
"navigation",
"reels tab",
@@ -44,6 +52,14 @@ NAV_INTENT_KEYWORDS = [
"followers",
]
NAV_TAB_KEYWORDS = [
"reels tab",
"profile tab",
"home tab",
"explore tab",
"message tab",
]
# Cache files
MEMORY_FILE = "telepathic_memory.json"
BLACKLIST_FILE = "telepathic_blacklist.json"
@@ -410,7 +426,7 @@ class TelepathicEngine:
# 3. Reject nodes in the Navigation Bar zone (bottom 6% - adjusted for accuracy)
# UNLESS the intent is explicitly about navigation tabs, profile stats, OR popup modals
nav_keywords = NAV_INTENT_KEYWORDS
nav_keywords = NAV_ZONE_BYPASS_KEYWORDS
modal_keywords = ["dismiss", "ok", "cancel", "accept", "allow", "deny", "action", "obstacle", "popup"]
low_intent = intent_description.lower()
@@ -431,12 +447,11 @@ class TelepathicEngine:
# STRICT NAVIGATION TAB ENFORCEMENT
# Navigation tabs MUST be at the bottom (>= 0.92 of screen height).
# We reject any navigation tab hallucinated in the middle of the screen.
if is_nav_intent and node.get("y", 0) < screen_height * 0.92:
# We must be careful: intent could be "navigate to profile" which means click username
# So ONLY block if the intent explicitly says "tab".
if "tab" in low_intent:
# Silently filter non-bottom elements for navigation intents to prevent log spam
return False
# STRICT TAB ENFORCEMENT: Only actual tab intents must be at the bottom.
is_tab_intent = any(k in low_intent for k in NAV_TAB_KEYWORDS)
if is_tab_intent and node.get("y", 0) < screen_height * 0.92:
# Silently filter non-bottom elements for tab intents to prevent log spam
return False
# STRICT STORY TRAY ENFORCEMENT
# Story rings MUST be at the top of the screen (y < 30% of screen height).
@@ -685,7 +700,7 @@ class TelepathicEngine:
# - Navigation intents: Require 100% exact match to avoid feed-cross-talk
# - Short intents (1-2 words): Require at least 50% hit (0.45)
# - Longer intents: Require 75% to avoid false matches on noisy screens.
is_nav_intent = any(k in intent_lower for k in NAV_INTENT_KEYWORDS)
is_nav_intent = any(k in intent_lower for k in NAV_ZONE_BYPASS_KEYWORDS)
if is_nav_intent:
threshold = 1.0
else:
@@ -939,7 +954,7 @@ class TelepathicEngine:
# If a bottom sheet or dialog is active, it likely obscures the main navigation tabs.
# We rely strictly on the SAE (Situational Awareness Engine) for 100% autonomous detection
# (via Qdrant cache or LLM reasoning) instead of hardcoded resource IDs.
is_nav_intent = any(k in intent_lower for k in NAV_INTENT_KEYWORDS)
is_nav_intent = any(k in intent_lower for k in NAV_ZONE_BYPASS_KEYWORDS)
if is_nav_intent and device is not None:
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
@@ -2080,15 +2095,14 @@ class TelepathicEngine:
)
return None
is_nav_intent = any(k in intent.lower() for k in NAV_INTENT_KEYWORDS)
is_nav_intent = any(k in intent.lower() for k in NAV_ZONE_BYPASS_KEYWORDS)
is_tab_intent = any(k in intent.lower() for k in NAV_TAB_KEYWORDS)
# NAVIGATION TAB ENFORCEMENT:
# Real navigation tabs (Home, Search, Reels, Store, Profile) are ALWAYS in the bottom zone.
# If the bot is looking for a tab, forbid results that are too high up (likely hallucinations on comments/feed).
# IMPORTANT: Only enforce for intents explicitly containing "tab".
# "following"/"follower" are profile stats at the TOP of the screen, not tabs.
# This aligns with _structural_sanity_check which has the same "tab" guard.
if is_nav_intent and "tab" in intent.lower():
# If the bot is looking for a tab, forbid results that are too high up.
# "following"/"follower" are profile stats at the TOP — not tabs.
if is_tab_intent:
if match.get("y", 0) < screen_height * 0.92:
logger.warning(
f"🛡️ [Structural Guard] VLM hallucinated a navigation tab '{intent}' "