mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: enforce zero-maintenance autonomous navigation by purging hardcoded string searches and localized translations; harden VLM perception via robust JSON fallback; fix OTHER_PROFILE topological routing

Browse Source
This commit is contained in:
Marc Mintel
2026-05-03 23:28:00 +02:00
parent 4b645c6fb2
commit c641204a6b
19 changed files with 455 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/e2e/test_behavior_story_view.py
View File

@@ -41,7 +41,7 @@ def test_story_view_clicks_story_ring(make_real_device_with_image, e2e_configs,
# 3. goap verification post-click (xml_after)
# 4. Fallbacks/extras (xml_after, xml_after)
device = make_real_device_with_image(
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_before, xml_after, xml_after, xml_after]
"tests/fixtures/home_feed_with_ad.jpg", [xml_before, xml_after, xml_after, xml_after]
)
from GramAddict.core.session_state import SessionState

32
tests/e2e/test_sae_foreign_app_fastpath.py
View File

@@ -54,3 +54,35 @@ def test_sae_fast_path_handles_known_foreign_packages():
result = sae.perceive(xml)
assert result == expected, f"Package {pkg} was classified as {result.value}, expected {expected.value}!"
def test_sae_ensure_clear_screen_escapes_foreign_app_without_llm():
"""
If SAE perceives a FOREIGN_APP, ensure_clear_screen must immediately
execute a kill_foreign_apps action WITHOUT consulting the LLM.
"""
xml_foreign = load_fixture_xml("play_store_from_story_link.xml")
xml_normal = """<?xml version="1.0" encoding="utf-8"?>
<hierarchy rotation="0">
<node class="android.widget.FrameLayout" package="com.instagram.android"
bounds="[0,0][1080,2400]">
<node text="Instagram" class="android.widget.TextView" />
</node>
</hierarchy>"""
# device returns foreign first, then normal on next dump
device = E2EDeviceStub([xml_foreign, xml_normal])
SituationalAwarenessEngine.reset()
sae = SituationalAwarenessEngine(device)
# Patch _plan_escape_via_llm to raise an error if called!
def _mock_plan(*args, **kwargs):
raise AssertionError("LLM was called for FOREIGN_APP escape! This should be an O(1) fast-path.")
sae._plan_escape_via_llm = _mock_plan
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True, "SAE failed to clear the foreign app screen"
assert "com.instagram.android" in device.app_starts, "SAE did not attempt to restart Instagram"

159
tests/e2e/test_system_goap_navigation.py
View File

@@ -24,11 +24,11 @@ import pytest
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
from GramAddict.core.screen_topology import ScreenTopology
# ═══════════════════════════════════════════════════════════════════════
# Helpers
# ═══════════════════════════════════════════════════════════════════════
def _xml_with_ids(*resource_ids, selected_tab=None, texts=None, descs=None):
"""Build a minimal XML dump with given resource IDs."""
nodes = []
@@ -40,13 +40,13 @@ def _xml_with_ids(*resource_ids, selected_tab=None, texts=None, descs=None):
f'text="" content-desc="" clickable="true" selected="{selected}" '
f'bounds="[0,0][100,100]" />'
)
for text in (texts or []):
for text in texts or []:
nodes.append(
f'<node package="com.instagram.android" '
f'resource-id="" text="{text}" content-desc="" '
f'clickable="false" selected="false" bounds="[0,0][100,100]" />'
)
for desc in (descs or []):
for desc in descs or []:
nodes.append(
f'<node package="com.instagram.android" '
f'resource-id="" text="" content-desc="{desc}" '
@@ -71,20 +71,17 @@ class TestScreenIdentification:
self.si = ScreenIdentity(bot_username="testbot")
def test_home_feed_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="feed_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.HOME_FEED
def test_explore_grid_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="search_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="search_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.EXPLORE_GRID
def test_reels_feed_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
selected_tab="clips_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", selected_tab="clips_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.REELS_FEED
@@ -95,16 +92,15 @@ class TestScreenIdentification:
assert result["screen_type"] == ScreenType.REELS_FEED
def test_own_profile_from_selected_tab(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
"profile_header_container",
selected_tab="profile_tab")
xml = _xml_with_ids(
"feed_tab", "search_tab", "clips_tab", "profile_tab", "profile_header_container", selected_tab="profile_tab"
)
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.OWN_PROFILE
def test_other_profile_from_header_without_tab(self):
"""Other profile has header but profile_tab is NOT selected."""
xml = _xml_with_ids("profile_header_container", "feed_tab",
selected_tab="feed_tab")
xml = _xml_with_ids("profile_header_container", "feed_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.OTHER_PROFILE
@@ -146,9 +142,13 @@ class TestScreenIdentification:
def test_home_feed_has_feed_markers_with_action_bar(self):
"""HOME_FEED has row_feed_* markers AND main_feed_action_bar."""
xml = _xml_with_ids("row_feed_button_like", "row_feed_photo_profile_name",
"main_feed_action_bar", "feed_tab",
selected_tab="feed_tab")
xml = _xml_with_ids(
"row_feed_button_like",
"row_feed_photo_profile_name",
"main_feed_action_bar",
"feed_tab",
selected_tab="feed_tab",
)
result = self.si.identify(xml)
assert result["screen_type"] == ScreenType.HOME_FEED
@@ -177,8 +177,7 @@ class TestAvailableActions:
self.si = ScreenIdentity(bot_username="testbot")
def test_home_feed_has_all_tabs(self):
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab",
"direct_tab", selected_tab="feed_tab")
xml = _xml_with_ids("feed_tab", "search_tab", "clips_tab", "profile_tab", "direct_tab", selected_tab="feed_tab")
result = self.si.identify(xml)
actions = result["available_actions"]
assert "tap home tab" in actions
@@ -194,9 +193,7 @@ class TestAvailableActions:
assert "tap first post" in actions
def test_profile_has_following_list(self):
xml = _xml_with_ids("profile_header_container", "profile_tab",
selected_tab="profile_tab",
descs=["following"])
xml = _xml_with_ids("profile_header_container", "profile_tab", selected_tab="profile_tab", descs=["following"])
result = self.si.identify(xml)
actions = result["available_actions"]
assert "tap following list" in actions
@@ -246,8 +243,8 @@ class TestHDMapRouting:
assert route == []
def test_unreachable_returns_none(self):
"""POST_DETAIL has no outgoing edges → can't reach other screens."""
route = ScreenTopology.find_route(ScreenType.POST_DETAIL, ScreenType.DM_INBOX)
"""MODAL has no outgoing edges → can't reach other screens."""
route = ScreenTopology.find_route(ScreenType.MODAL, ScreenType.DM_INBOX)
assert route is None
def test_masked_edge_makes_route_none(self):
@@ -281,9 +278,7 @@ class TestHDMapRouting:
ScreenType.REELS_FEED,
avoid_actions=all_reels_actions,
)
assert route is None, (
f"Expected None (unreachable) but got route: {route}"
)
assert route is None, f"Expected None (unreachable) but got route: {route}"
# ═══════════════════════════════════════════════════════════════════════
@@ -318,10 +313,20 @@ class TestGoalMapping:
("comment on the post", None),
],
ids=[
"home_feed", "home_short", "explore", "explore_full",
"reels", "profile", "learn_profile", "messages",
"following_list", "post", "other_profile",
"like_non_nav", "follow_non_nav", "comment_non_nav",
"home_feed",
"home_short",
"explore",
"explore_full",
"reels",
"profile",
"learn_profile",
"messages",
"following_list",
"post",
"other_profile",
"like_non_nav",
"follow_non_nav",
"comment_non_nav",
],
)
def test_goal_mapping(self, goal, expected_screen):
@@ -381,14 +386,13 @@ class TestActionMasking:
avoid_actions = {"tap reels tab"}
# Step 1: Check if route exists without masking
route_clean = ScreenTopology.find_route(
ScreenType.HOME_FEED, ScreenType.REELS_FEED
)
route_clean = ScreenTopology.find_route(ScreenType.HOME_FEED, ScreenType.REELS_FEED)
assert route_clean is not None, "Route should exist without masking"
# Step 2: Check if route exists WITH masking
route_masked = ScreenTopology.find_route(
ScreenType.HOME_FEED, ScreenType.REELS_FEED,
ScreenType.HOME_FEED,
ScreenType.REELS_FEED,
avoid_actions=avoid_actions,
)
@@ -415,57 +419,40 @@ class TestGoalAchievement:
def setup_method(self):
from GramAddict.core.navigation.planner import GoalPlanner
self.planner = GoalPlanner(username="testbot")
def test_open_explore_achieved_on_explore(self):
assert self.planner._is_goal_achieved(
"open explore", ScreenType.EXPLORE_GRID, {}
) is True
assert self.planner._is_goal_achieved("open explore", ScreenType.EXPLORE_GRID, {}) is True
def test_open_explore_not_achieved_on_home(self):
assert self.planner._is_goal_achieved(
"open explore", ScreenType.HOME_FEED, {}
) is False
assert self.planner._is_goal_achieved("open explore", ScreenType.HOME_FEED, {}) is False
def test_open_reels_achieved_on_reels(self):
assert self.planner._is_goal_achieved(
"open reels", ScreenType.REELS_FEED, {}
) is True
assert self.planner._is_goal_achieved("open reels", ScreenType.REELS_FEED, {}) is True
def test_open_reels_not_achieved_on_explore(self):
"""
Production bug 2026-05-02: Bot tapped 'reels tab' but
landed on EXPLORE_GRID. Goal must NOT be achieved.
"""
assert self.planner._is_goal_achieved(
"open reels", ScreenType.EXPLORE_GRID, {}
) is False
assert self.planner._is_goal_achieved("open reels", ScreenType.EXPLORE_GRID, {}) is False
def test_like_achieved_when_context_is_liked(self):
assert self.planner._is_goal_achieved(
"like a post", ScreenType.POST_DETAIL, {"is_liked": True}
) is True
assert self.planner._is_goal_achieved("like a post", ScreenType.POST_DETAIL, {"is_liked": True}) is True
def test_like_not_achieved_when_not_liked(self):
assert self.planner._is_goal_achieved(
"like a post", ScreenType.POST_DETAIL, {"is_liked": False}
) is False
assert self.planner._is_goal_achieved("like a post", ScreenType.POST_DETAIL, {"is_liked": False}) is False
def test_view_profile_achieved_on_own_profile(self):
assert self.planner._is_goal_achieved(
"view profile", ScreenType.OWN_PROFILE, {}
) is True
assert self.planner._is_goal_achieved("view profile", ScreenType.OWN_PROFILE, {}) is True
def test_view_profile_achieved_on_other_profile(self):
assert self.planner._is_goal_achieved(
"view profile", ScreenType.OTHER_PROFILE, {}
) is True
assert self.planner._is_goal_achieved("view profile", ScreenType.OTHER_PROFILE, {}) is True
def test_non_navigation_goal_not_achieved_by_screen(self):
"""Goals like 'follow the user' are NOT achieved by just being on a screen."""
assert self.planner._is_goal_achieved(
"follow the user", ScreenType.OTHER_PROFILE, {}
) is False
assert self.planner._is_goal_achieved("follow the user", ScreenType.OTHER_PROFILE, {}) is False
# ═══════════════════════════════════════════════════════════════════════
@@ -482,28 +469,18 @@ class TestStructuralActionProtection:
"""Tests that HD Map actions are never permanently poisoned."""
def test_tap_reels_tab_is_structural_on_home(self):
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "tap reels tab"
) is True
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "tap reels tab") is True
def test_tap_profile_tab_is_structural_on_explore(self):
assert ScreenTopology.is_structural_action(
ScreenType.EXPLORE_GRID, "tap profile tab"
) is True
assert ScreenTopology.is_structural_action(ScreenType.EXPLORE_GRID, "tap profile tab") is True
def test_random_action_is_not_structural(self):
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "like this post"
) is False
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "like this post") is False
def test_structural_on_wrong_screen_is_false(self):
"""'tap following list' is structural on OWN_PROFILE, not on HOME_FEED."""
assert ScreenTopology.is_structural_action(
ScreenType.HOME_FEED, "tap following list"
) is False
assert ScreenTopology.is_structural_action(
ScreenType.OWN_PROFILE, "tap following list"
) is True
assert ScreenTopology.is_structural_action(ScreenType.HOME_FEED, "tap following list") is False
assert ScreenTopology.is_structural_action(ScreenType.OWN_PROFILE, "tap following list") is True
# ═══════════════════════════════════════════════════════════════════════
@@ -520,15 +497,11 @@ class TestStepValidation:
"""Tests expected_screen_for_action validation."""
def test_tap_reels_tab_from_home_expects_reels(self):
expected = ScreenTopology.expected_screen_for_action(
"tap reels tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap reels tab", ScreenType.HOME_FEED)
assert expected == ScreenType.REELS_FEED
def test_tap_explore_tab_from_home_expects_explore(self):
expected = ScreenTopology.expected_screen_for_action(
"tap explore tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap explore tab", ScreenType.HOME_FEED)
assert expected == ScreenType.EXPLORE_GRID
def test_tap_reels_tab_landing_on_explore_is_failure(self):
@@ -536,20 +509,22 @@ class TestStepValidation:
Production bug 2026-05-02: Bot tapped 'reels tab' but
landed on EXPLORE_GRID. This must be detected as a failure.
"""
expected = ScreenTopology.expected_screen_for_action(
"tap reels tab", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("tap reels tab", ScreenType.HOME_FEED)
actual = ScreenType.EXPLORE_GRID
assert expected != actual, "Reels tab landing on Explore must be a mismatch!"
def test_unknown_action_returns_none(self):
expected = ScreenTopology.expected_screen_for_action(
"like this post", ScreenType.HOME_FEED
)
expected = ScreenTopology.expected_screen_for_action("like this post", ScreenType.HOME_FEED)
assert expected is None
def test_press_back_from_follow_list_expects_profile(self):
expected = ScreenTopology.expected_screen_for_action(
"press back", ScreenType.FOLLOW_LIST
)
expected = ScreenTopology.expected_screen_for_action("press back", ScreenType.FOLLOW_LIST)
assert expected == ScreenType.OWN_PROFILE
def test_other_profile_to_home_feed_routing_uses_back_press(self):
# We removed 'tap home tab' from OTHER_PROFILE because it doesn't exist.
route = ScreenTopology.find_route(ScreenType.OTHER_PROFILE, ScreenType.HOME_FEED)
assert route is not None
assert len(route) == 1
assert route[0][0] == "press back"
assert route[0][1] == ScreenType.HOME_FEED

22
tests/e2e/test_system_llm_perception.py
View File

@@ -915,3 +915,25 @@ class TestVerifySuccessStructuralDelta:
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
assert result is False
def test_semantic_evaluator_malformed_json_fallback(self):
"""TDD Test to ensure malformed JSON without closing braces is handled automatically without regex."""
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
evaluator = SemanticEvaluator()
# Override the VLM query to return a truncated JSON string (like an LLM out of tokens)
evaluator._query_vlm = (
lambda prompt, img: '{\n "should_like": true,\n "should_comment": false,\n "is_ad": false'
)
class MockDevice:
def get_screenshot_b64(self):
return "dummy_b64"
result = evaluator.evaluate_post_vibe(MockDevice(), ["test"])
assert result is not None
assert result["should_like"] is True
assert result["should_comment"] is False
assert result["is_ad"] is False