mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: enforce zero-maintenance autonomous navigation by purging hardcoded string searches and localized translations; harden VLM perception via robust JSON fallback; fix OTHER_PROFILE topological routing

Browse Source
This commit is contained in:
Marc Mintel
2026-05-03 23:28:00 +02:00
parent 4b645c6fb2
commit c641204a6b
19 changed files with 455 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
tests/core/test_action_memory_fast_path.py Normal file
View File

@@ -0,0 +1,56 @@
from GramAddict.core.perception.action_memory import ActionMemory
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
def test_screen_identity_detects_other_profile_with_missing_header_container():
"""
RED: ScreenIdentity used to misclassify OTHER_PROFILE as UNKNOWN or POST_DETAIL
if `profile_header_container` was missing, even though `row_profile_header_imageview`
was present.
GREEN: We added row_profile_header_imageview and profile_tabs_container.
"""
identity = ScreenIdentity(bot_username="my_bot")
# Simulate a profile screen that is missing the main container but has the imageview
xml_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy>
<node package="com.instagram.android" resource-id="com.instagram.android:id/row_profile_header_imageview" bounds="[0,0][100,100]" clickable="true"/>
<node package="com.instagram.android" resource-id="com.instagram.android:id/profile_tabs_container" bounds="[0,200][1080,300]"/>
<node package="com.instagram.android" resource-id="com.instagram.android:id/action_bar_title" text="justkay"/>
</hierarchy>
"""
result = identity.identify(xml_dump)
assert result["screen_type"] == ScreenType.OTHER_PROFILE
def test_action_memory_verifies_profile_navigation_in_o1_without_vlm(monkeypatch):
"""
RED: ActionMemory.verify_success used to fallback to VLM because navigating to a profile
caused a huge delta, but there was no explicit fast-path for 'profile', causing it
to hit `confidence < 0.95` and invoke `evaluator._query_vlm`.
GREEN: It now structurally verifies `profile_header_container` instantly.
"""
memory = ActionMemory()
class DummyDevice:
def get_screenshot_b64(self):
raise Exception("VLM SHOULD NOT BE CALLED!")
device = DummyDevice()
intent = "tap post username"
pre_xml = "<hierarchy><node/></hierarchy>"
# Post XML contains profile_header_container!
post_xml = "<hierarchy><node resource-id='com.instagram.android:id/profile_header_container'/></hierarchy>"
# If the fast-path works, it will return True instantly and NOT call get_screenshot_b64
success = memory.verify_success(
intent=intent,
pre_click_xml=pre_xml,
post_click_xml=post_xml,
device=device,
confidence=0.0, # low confidence triggers VLM fallback if fast-path is missing
)
assert success is True

51
tests/core/test_intent_resolver_multilingual.py Normal file
View File

@@ -0,0 +1,51 @@
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialNode
def test_semantic_guard_allows_multilingual_follow_button():
"""
RED: The IntentResolver's Semantic Guard used to hard-filter for EXACT quotes.
If the plugin requested "tap 'Follow' button", but the UI was in German ("Abonnieren"),
the Semantic Guard would block it, causing the bot to never follow anyone.
GREEN: We added multilingual equivalents to the Semantic Guard logic.
"""
resolver = IntentResolver()
intent = "tap 'Follow' button"
# Create a node that represents a German follow button
german_node = SpatialNode(
resource_id="com.instagram.android:id/profile_header_follow_button",
content_desc="Abonnieren",
text="Abonnieren",
bounds=(0, 0, 100, 100),
)
# Run the resolver without a device (forces semantic/structural resolution, bypasses VLM)
result = resolver.resolve(intent, candidates=[german_node], device=None, screen_height=2000)
assert result is not None
assert result.text == "Abonnieren"
def test_semantic_guard_allows_multilingual_following_button():
"""
Ensures that "tap 'Following' button" resolves correctly for German UIs
("Abonniert", "Gefolgt", "Angefragt").
"""
resolver = IntentResolver()
intent = "tap 'Following' button"
# Create a node that represents a German "Following" button
german_node = SpatialNode(
resource_id="com.instagram.android:id/profile_header_follow_button",
content_desc="Abonniert",
text="Abonniert",
bounds=(0, 0, 100, 100),
)
result = resolver.resolve(intent, candidates=[german_node], device=None, screen_height=2000)
assert result is not None
assert result.text == "Abonniert"

2
tests/e2e/test_behavior_story_view.py
View File

@@ -41,7 +41,7 @@ def test_story_view_clicks_story_ring(make_real_device_with_image, e2e_configs,
# 3. goap verification post-click (xml_after)
# 4. Fallbacks/extras (xml_after, xml_after)
device = make_real_device_with_image(
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_before, xml_after, xml_after, xml_after]
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_after, xml_after, xml_after]
)
from GramAddict.core.session_state import SessionState

32
tests/e2e/test_sae_foreign_app_fastpath.py
View File

@@ -54,3 +54,35 @@ def test_sae_fast_path_handles_known_foreign_packages():
result = sae.perceive(xml)
assert result == expected, f"Package {pkg} was classified as {result.value}, expected {expected.value}!"
def test_sae_ensure_clear_screen_escapes_foreign_app_without_llm():
"""
If SAE perceives a FOREIGN_APP, ensure_clear_screen must immediately
execute a kill_foreign_apps action WITHOUT consulting the LLM.
"""
xml_foreign = load_fixture_xml("play_store_from_story_link.xml")
xml_normal = """<?xml version="1.0" encoding="utf-8"?>
<hierarchy rotation="0">
<node class="android.widget.FrameLayout" package="com.instagram.android"
bounds="[0,0][1080,2400]">
<node text="Instagram" class="android.widget.TextView" />
</node>
</hierarchy>"""
# device returns foreign first, then normal on next dump
device = E2EDeviceStub([xml_foreign, xml_normal])
SituationalAwarenessEngine.reset()
sae = SituationalAwarenessEngine(device)
# Patch _plan_escape_via_llm to raise an error if called!
def _mock_plan(*args, **kwargs):
raise AssertionError("LLM was called for FOREIGN_APP escape! This should be an O(1) fast-path.")
sae._plan_escape_via_llm = _mock_plan
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True, "SAE failed to clear the foreign app screen"
assert "com.instagram.android" in device.app_starts, "SAE did not attempt to restart Instagram"

159
tests/e2e/test_system_goap_navigation.py
View File

@@ -24,11 +24,11 @@ import pytest
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
from GramAddict.core.screen_topology import ScreenTopology
# ═══════════════════════════════════════════════════════════════════════
# Helpers
# ═══════════════════════════════════════════════════════════════════════
def _xml_with_ids(*resource_ids, selected_tab=None, texts=None, descs=None):
"""Build a minimal XML dump with given resource IDs."""
nodes = []
@@ -40,13 +40,13 @@ def _xml_with_ids(*resource_ids, selected_tab=None, texts=None, descs=None):
f'text="" content-desc="" clickable="true" selected="{selected}" '
f'bounds="[0,0][100,100]" />'
)
for text in (texts or []):
for text in texts or []:
nodes.append(
f'<node package="com.instagram.android" '
f'resource-id="" text="{text}" content-desc="" '
f'clickable="false" selected="false" bounds="[0,0][100,100]" />'
)
for desc in (descs or []):
for desc in descs or []:
nodes.append(
f'<node package="com.instagram.android" '
f'resource-id="" text="" content-desc="{desc}" '
@@ -71,20 +71,17 @@ class TestScreenIdentification:
self.si = ScreenIdentity(bot_username="testbot")
def test_home_feed_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="feed_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.HOME_FEED
def test_explore_grid_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="search_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="search_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.EXPLORE_GRID
def test_reels_feed_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="clips_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="clips_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.REELS_FEED
@@ -95,16 +92,15 @@ class TestScreenIdentification:
assert result["screen_type"] == ScreenType.REELS_FEED
def test_own_profile_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
"profile_header_container",
selected_tab="profile_tab")
xml = _xml_with_ids(
"feed_tab", "search_tab", "clips_tab", "profile_tab", "profile_header_container", selected_tab="profile_tab"
)
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.OWN_PROFILE
def test_other_profile_from_header_without_tab(self):
"""Other profile has header but profile_tab is NOT selected."""
xml = _xml_with_ids("profile_header_container", "feed_tab",
selected_tab="feed_tab")
xml = _xml_with_ids("profile_header_container", "feed_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.OTHER_PROFILE
@@ -146,9 +142,13 @@ class TestScreenIdentification:
def test_home_feed_has_feed_markers_with_action_bar(self):
"""HOME_FEED has row_feed_* markers AND main_feed_action_bar."""
xml = _xml_with_ids("row_feed_button_like", "row_feed_photo_profile_name",
"main_feed_action_bar", "feed_tab",
selected_tab="feed_tab")
xml = _xml_with_ids(
"row_feed_button_like",
"row_feed_photo_profile_name",
"main_feed_action_bar",
"feed_tab",
selected_tab="feed_tab",
)
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.HOME_FEED
@@ -177,8 +177,7 @@ class TestAvailableActions:
self.si = ScreenIdentity(bot_username="testbot")
def test_home_feed_has_all_tabs(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
"direct_tab", selected_tab="feed_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
actions = result["available_actions"]
assert "tap home tab" in actions
@@ -194,9 +193,7 @@ class TestAvailableActions:
assert "tap first post" in actions
def test_profile_has_following_list(self):
xml = _xml_with_ids("profile_header_container", "profile_tab",
selected_tab="profile_tab",
descs=["following"])
xml = _xml_with_ids("profile_header_container", "profile_tab", selected_tab="profile_tab", descs=["following"])
result = self.si.identify(xml)
actions = result["available_actions"]
assert "tap following list" in actions
@@ -246,8 +243,8 @@ class TestHDMapRouting:
assert route == []
def test_unreachable_returns_none(self):
"""POST_DETAIL has no outgoing edges → can't reach other screens."""
route = ScreenTopology.find_route(ScreenType.POST_DETAIL, ScreenType.DM_INBOX)
"""MODAL has no outgoing edges → can't reach other screens."""
route = ScreenTopology.find_route(ScreenType.MODAL, ScreenType.DM_INBOX)
assert route is None
def test_masked_edge_makes_route_none(self):
@@ -281,9 +278,7 @@ class TestHDMapRouting:
ScreenType.REELS_FEED,
avoid_actions=all_reels_actions,
)
assert route is None, (
f"Expected None (unreachable) but got route: {route}"
)
assert route is None, f"Expected None (unreachable) but got route: {route}"
# ═══════════════════════════════════════════════════════════════════════
@@ -318,10 +313,20 @@ class TestGoalMapping:
("comment on the post", None),
],
ids=[
"home_feed", "home_short", "explore", "explore_full",
"reels", "profile", "learn_profile", "messages",
"following_list", "post", "other_profile",
"like_non_nav", "follow_non_nav", "comment_non_nav",
"home_feed",
"home_short",
"explore",
"explore_full",
"reels",
"profile",
"learn_profile",
"messages",
"following_list",
"post",
"other_profile",
"like_non_nav",
"follow_non_nav",
"comment_non_nav",
],
)
def test_goal_mapping(self, goal, expected_screen):
@@ -381,14 +386,13 @@ class TestActionMasking:
avoid_actions = {"tap reels tab"}
# Step 1: Check if route exists without masking
route_clean = ScreenTopology.find_route(
ScreenType.HOME_FEED, ScreenType.REELS_FEED
)
route_clean = ScreenTopology.find_route(ScreenType.HOME_FEED, ScreenType.REELS_FEED)
assert route_clean is not None, "Route should exist without masking"
# Step 2: Check if route exists WITH masking
route_masked = ScreenTopology.find_route(
ScreenType.HOME_FEED, ScreenType.REELS_FEED,
ScreenType.HOME_FEED,
ScreenType.REELS_FEED,
avoid_actions=avoid_actions,
)
@@ -415,57 +419,40 @@ class TestGoalAchievement:
def setup_method(self):
from GramAddict.core.navigation.planner import GoalPlanner
self.planner = GoalPlanner(username="testbot")
def test_open_explore_achieved_on_explore(self):
assert self.planner._is_goal_achieved(
"open explore", ScreenType.EXPLORE_GRID, {}
) is True
assert self.planner._is_goal_achieved("open explore", ScreenType.EXPLORE_GRID, {}) is True
def test_open_explore_not_achieved_on_home(self):
assert self.planner._is_goal_achieved(
"open explore", ScreenType.HOME_FEED, {}
) is False
assert self.planner._is_goal_achieved("open explore", ScreenType.HOME_FEED, {}) is False
def test_open_reels_achieved_on_reels(self):
assert self.planner._is_goal_achieved(
"open reels", ScreenType.REELS_FEED, {}
) is True
assert self.planner._is_goal_achieved("open reels", ScreenType.REELS_FEED, {}) is True
def test_open_reels_not_achieved_on_explore(self):
"""
Production bug 2026-05-02: Bot tapped 'reels tab' but
landed on EXPLORE_GRID. Goal must NOT be achieved.
"""
assert self.planner._is_goal_achieved(
"open reels", ScreenType.EXPLORE_GRID, {}
) is False
assert self.planner._is_goal_achieved("open reels", ScreenType.EXPLORE_GRID, {}) is False
def test_like_achieved_when_context_is_liked(self):
assert self.planner._is_goal_achieved(
"like a post", ScreenType.POST_DETAIL, {"is_liked": True}
) is True
assert self.planner._is_goal_achieved("like a post", ScreenType.POST_DETAIL, {"is_liked": True}) is True
def test_like_not_achieved_when_not_liked(self):
assert self.planner._is_goal_achieved(
"like a post", ScreenType.POST_DETAIL, {"is_liked": False}
) is False
assert self.planner._is_goal_achieved("like a post", ScreenType.POST_DETAIL, {"is_liked": False}) is False
def test_view_profile_achieved_on_own_profile(self):
assert self.planner._is_goal_achieved(
"view profile", ScreenType.OWN_PROFILE, {}
) is True
assert self.planner._is_goal_achieved("view profile", ScreenType.OWN_PROFILE, {}) is True
def test_view_profile_achieved_on_other_profile(self):
assert self.planner._is_goal_achieved(
"view profile", ScreenType.OTHER_PROFILE, {}
) is True
assert self.planner._is_goal_achieved("view profile", ScreenType.OTHER_PROFILE, {}) is True
def test_non_navigation_goal_not_achieved_by_screen(self):
"""Goals like 'follow the user' are NOT achieved by just being on a screen."""
assert self.planner._is_goal_achieved(
"follow the user", ScreenType.OTHER_PROFILE, {}
) is False
assert self.planner._is_goal_achieved("follow the user", ScreenType.OTHER_PROFILE, {}) is False
# ═══════════════════════════════════════════════════════════════════════
@@ -482,28 +469,18 @@ class TestStructuralActionProtection:
"""Tests that HD Map actions are never permanently poisoned."""
def test_tap_reels_tab_is_structural_on_home(self):
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "tap reels tab"
) is True
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "tap reels tab") is True
def test_tap_profile_tab_is_structural_on_explore(self):
assert ScreenTopology.is_structural_action(
ScreenType.EXPLORE_GRID, "tap profile tab"
) is True
assert ScreenTopology.is_structural_action(ScreenType.EXPLORE_GRID, "tap profile tab") is True
def test_random_action_is_not_structural(self):
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "like this post"
) is False
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "like this post") is False
def test_structural_on_wrong_screen_is_false(self):
"""'tap following list' is structural on OWN_PROFILE, not on HOME_FEED."""
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "tap following list"
) is False
assert ScreenTopology.is_structural_action(
ScreenType.OWN_PROFILE, "tap following list"
) is True
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "tap following list") is False
assert ScreenTopology.is_structural_action(ScreenType.OWN_PROFILE, "tap following list") is True
# ═══════════════════════════════════════════════════════════════════════
@@ -520,15 +497,11 @@ class TestStepValidation:
"""Tests expected_screen_for_action validation."""
def test_tap_reels_tab_from_home_expects_reels(self):
expected = ScreenTopology.expected_screen_for_action(
"tap reels tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap reels tab", ScreenType.HOME_FEED)
assert expected == ScreenType.REELS_FEED
def test_tap_explore_tab_from_home_expects_explore(self):
expected = ScreenTopology.expected_screen_for_action(
"tap explore tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap explore tab", ScreenType.HOME_FEED)
assert expected == ScreenType.EXPLORE_GRID
def test_tap_reels_tab_landing_on_explore_is_failure(self):
@@ -536,20 +509,22 @@ class TestStepValidation:
Production bug 2026-05-02: Bot tapped 'reels tab' but
landed on EXPLORE_GRID. This must be detected as a failure.
"""
expected = ScreenTopology.expected_screen_for_action(
"tap reels tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap reels tab", ScreenType.HOME_FEED)
actual = ScreenType.EXPLORE_GRID
assert expected != actual, "Reels tab landing on Explore must be a mismatch!"
def test_unknown_action_returns_none(self):
expected = ScreenTopology.expected_screen_for_action(
"like this post", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("like this post", ScreenType.HOME_FEED)
assert expected is None
def test_press_back_from_follow_list_expects_profile(self):
expected = ScreenTopology.expected_screen_for_action(
"press back", ScreenType.FOLLOW_LIST
)
expected = ScreenTopology.expected_screen_for_action("press back", ScreenType.FOLLOW_LIST)
assert expected == ScreenType.OWN_PROFILE
def test_other_profile_to_home_feed_routing_uses_back_press(self):
# We removed 'tap home tab' from OTHER_PROFILE because it doesn't exist.
route = ScreenTopology.find_route(ScreenType.OTHER_PROFILE, ScreenType.HOME_FEED)
assert route is not None
assert len(route) == 1
assert route[0][0] == "press back"
assert route[0][1] == ScreenType.HOME_FEED

22
tests/e2e/test_system_llm_perception.py
View File

@@ -915,3 +915,25 @@ class TestVerifySuccessStructuralDelta:
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
assert result is False
def test_semantic_evaluator_malformed_json_fallback(self):
"""TDD Test to ensure malformed JSON without closing braces is handled automatically without regex."""
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
evaluator = SemanticEvaluator()
# Override the VLM query to return a truncated JSON string (like an LLM out of tokens)
evaluator._query_vlm = (
lambda prompt, img: '{\n "should_like": true,\n "should_comment": false,\n "is_ad": false'
)
class MockDevice:
def get_screenshot_b64(self):
return "dummy_b64"
result = evaluator.evaluate_post_vibe(MockDevice(), ["test"])
assert result is not None
assert result["should_like"] is True
assert result["should_comment"] is False
assert result["is_ad"] is False