mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

test: expand LLM perception suite to 80 tests — 3 new contracts

Browse Source 
CONTRACT 6: extract_json() — LLM Response Sanitizer (9 tests)
Previously COMPLETELY untested. Every single LLM response in the entire
system passes through this function. Tests:
- Clean JSON, markdown fences (json+plain), thinking block purge
- Text prefix before JSON, truncated JSON fuzzy recovery
- Garbage/empty/None resilience

CONTRACT 7: _visual_discovery Pre-Filters (8 tests)
The guards INSIDE _visual_discovery() that run BEFORE the VLM ever
sees the candidates. If they fail, the VLM gets garbage and hallucinates.
Tests:
- Area filter: tiny nodes (<200px²) and huge containers (>400000px²)
- SystemUI filter: Android system elements, notifications, battery
- Strict Button Guard: long text captions excluded for button intents
- Grid Item Guard: non-grid elements excluded, fallthrough on no match
- Author/Username Guard: nav tabs excluded for author intents

CONTRACT 8: verify_success Structural Delta (8 tests)
The post-click verification logic that decides if a click actually worked.
Tests:
- Toggle massive XML shift = navigation error (False)
- Toggle small diff = success (True)
- Toggle zero diff = inconclusive (None)
- Follow success with 'Requested' marker (private accounts)
- Follow success with German locale ('Abonniert')
- View post with clips_viewer marker (Reels)
- View post still on grid = inconclusive (None)
- Semantic gate blocks wrong toggle element (caption != like button)
This commit is contained in:
Marc Mintel
2026-05-02 22:10:57 +02:00
parent 1cc367697e
commit f6f15ebd9a
1 changed files with 365 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

365
tests/e2e/test_system_llm_perception.py
View File

@@ -515,3 +515,368 @@ class TestActionMemoryLifecycle:
"view a post", pre_click_xml="<node/>", post_click_xml=post_xml_success
)
assert result is True
# ═══════════════════════════════════════════════════════════════════════
# CONTRACT 6: extract_json() — LLM Response Sanitizer
# ═══════════════════════════════════════════════════════════════════════
#
# EVERY LLM response passes through this function. If it fails,
# the entire Brain/GOAP/Perception pipeline gets garbage data.
# Previously untested.
# ═══════════════════════════════════════════════════════════════════════
class TestExtractJson:
"""Validates extract_json handles every LLM output format."""
@pytest.mark.parametrize(
"raw_response,expected_keys",
[
# Clean JSON
('{"action": "scroll_down"}', ["action"]),
# Markdown code fences (VLM wraps JSON in ```)
('```json\n{"box": 3}\n```', ["box"]),
('```\n{"box": 3}\n```', ["box"]),
# Thinking blocks (reasoning models like qwen3.5)
(
'<think>I need to find the button...</think>{"box": 5}',
["box"],
),
# Text prefix before JSON
(
'Based on my analysis, the answer is: {"should_like": true}',
["should_like"],
),
# Truncated JSON — fuzzy recovery (only completed key-value pairs are salvaged)
('{"action": "scroll_down", "target": "feed', ["action"]),
# Pure garbage — must return None
("I don't know what to do", None),
("", None),
(None, None),
],
ids=[
"clean_json",
"markdown_json_fence",
"markdown_fence",
"thinking_block",
"text_prefix",
"truncated_recovery",
"garbage",
"empty",
"none",
],
)
def test_extract_json(self, raw_response, expected_keys):
from GramAddict.core.llm_provider import extract_json
result = extract_json(raw_response)
if expected_keys is None:
assert result is None, f"Expected None but got: {result}"
else:
assert result is not None, f"Expected JSON with keys {expected_keys} but got None"
import json as json_mod
parsed = json_mod.loads(result)
for key in expected_keys:
assert key in parsed, f"Expected key '{key}' in {parsed}"
# ═══════════════════════════════════════════════════════════════════════
# CONTRACT 7: _visual_discovery Pre-Filters
# ═══════════════════════════════════════════════════════════════════════
#
# These are the guards INSIDE _visual_discovery() that run BEFORE the
# VLM ever sees the candidates. If they fail, the VLM gets garbage
# candidates and hallucinates. Previously completely untested because
# they require a device object.
#
# We test them in ISOLATION by extracting the filter logic.
# ═══════════════════════════════════════════════════════════════════════
class TestVisualDiscoveryPreFilters:
"""Tests the pre-filtering logic that runs before the VLM sees candidates."""
# ── Area Filter ──
def test_area_filter_excludes_tiny_nodes(self):
"""Nodes with area <= 200 must be filtered (invisible dots)."""
tiny = _make_node(text="dot", bounds=(0, 0, 10, 10)) # area = 100
normal = _make_node(text="button", bounds=(0, 0, 50, 50)) # area = 2500
candidates = [tiny, normal]
filtered = [n for n in candidates if 200 < n.area < 400000]
assert len(filtered) == 1
assert filtered[0].text == "button"
def test_area_filter_excludes_huge_containers(self):
"""Nodes with area >= 400000 must be filtered (full-screen containers)."""
huge = _make_node(text="container", bounds=(0, 0, 1080, 2400)) # area = 2,592,000
normal = _make_node(text="button", bounds=(0, 0, 50, 50)) # area = 2500
candidates = [huge, normal]
filtered = [n for n in candidates if 200 < n.area < 400000]
assert len(filtered) == 1
assert filtered[0].text == "button"
# ── SystemUI Filter ──
def test_system_ui_excluded(self):
"""Android system UI elements must never reach the VLM."""
sys_node = _make_node(
resource_id="com.android.systemui:id/notification_icon",
content_desc="Battery 85 per cent",
)
app_node = _make_node(
resource_id="com.instagram.android:id/like_button",
content_desc="Like",
)
candidates = [sys_node, app_node]
filtered = [
n
for n in candidates
if "com.android.systemui" not in (n.resource_id or "")
and "notification:" not in (n.content_desc or "").lower()
and "per cent" not in (n.content_desc or "").lower()
]
assert len(filtered) == 1
assert filtered[0].content_desc == "Like"
# ── Strict Button Guard ──
def test_button_guard_filters_long_text_captions(self):
"""
When looking for a 'button', nodes with long text (captions,
comments) must be excluded to prevent VLM confusion.
"""
caption = _make_node(
text="This is a really long caption about my amazing vacation in Bali #travel #photography #blessed",
resource_id="caption_text",
)
button = _make_node(
text="Like",
resource_id="like_button",
)
candidates = [caption, button]
intent_lower = "like button"
# Replicate the exact guard from _visual_discovery
if "button" in intent_lower:
candidates = [n for n in candidates if len(n.text or "") < 40]
assert len(candidates) == 1
assert candidates[0].text == "Like"
def test_button_guard_preserves_short_text_buttons(self):
"""Short-text elements (actual buttons) must pass through."""
candidates = [
_make_node(text="Follow", resource_id="follow_btn"),
_make_node(text="Following", resource_id="following_btn"),
_make_node(text="Message", resource_id="msg_btn"),
]
intent_lower = "follow button"
if "button" in intent_lower:
candidates = [n for n in candidates if len(n.text or "") < 40]
assert len(candidates) == 3
# ── Grid Item Guard ──
def test_grid_guard_filters_non_grid_elements(self):
"""
When looking for 'first post', the guard must filter to actual
grid items (with 'row 1', 'photos by', 'reel by' in desc).
"""
search_tab = _make_node(content_desc="Search and explore", resource_id="search_tab")
grid_item = _make_node(
content_desc="6 photos by photographer_jane. Row 1, Column 1.",
resource_id="grid_card",
)
reel_item = _make_node(
content_desc="Reel by dancer_x. 1.2M views",
resource_id="reel_card",
)
candidates = [search_tab, grid_item, reel_item]
intent_lower = "first post"
if "first post" in intent_lower:
grid_candidates = []
for node in candidates:
desc = (node.content_desc or "").lower()
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
grid_candidates.append(node)
if grid_candidates:
candidates = grid_candidates
assert len(candidates) == 2
descs = [n.content_desc for n in candidates]
assert "Search and explore" not in descs
def test_grid_guard_fallthrough_when_no_grid_items(self):
"""If no grid items match, all candidates pass through (no crash)."""
candidates = [
_make_node(content_desc="Search"),
_make_node(content_desc="Home"),
]
intent_lower = "first post"
if "first post" in intent_lower:
grid_candidates = []
for node in candidates:
desc = (node.content_desc or "").lower()
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
grid_candidates.append(node)
if grid_candidates:
candidates = grid_candidates
# No grid items matched, so all candidates are kept
assert len(candidates) == 2
# ── Author/Username Guard ──
def test_author_guard_filters_nav_tabs(self):
"""
When looking for 'post author username', navigation tabs
(Profile, Home, Search, Reels) must be excluded.
"""
profile_tab = _make_node(
resource_id="com.instagram.android:id/profile_tab",
content_desc="Profile",
)
username = _make_node(
resource_id="com.instagram.android:id/row_feed_photo_profile_name",
text="photographer_jane",
content_desc="",
)
candidates = [profile_tab, username]
intent_lower = "post author username"
if "author" in intent_lower or "username" in intent_lower:
filtered = []
for node in candidates:
res_id = (node.resource_id or "").lower()
desc = (node.content_desc or "").lower()
if (
"tab" in res_id
or "navigation" in res_id
or "tabbar" in res_id
or desc in ["home", "search", "reels", "profile"]
):
continue
filtered.append(node)
candidates = filtered
assert len(candidates) == 1
assert candidates[0].text == "photographer_jane"
# ═══════════════════════════════════════════════════════════════════════
# CONTRACT 8: verify_success Structural Delta Logic
# ═══════════════════════════════════════════════════════════════════════
#
# After every click, verify_success() decides if the click worked.
# If the delta logic is wrong, the bot either:
# - Thinks failed clicks succeeded (ghost interactions)
# - Thinks successful clicks failed (infinite retry loops)
# Previously only tested for follow/view-post markers.
# ═══════════════════════════════════════════════════════════════════════
class TestVerifySuccessStructuralDelta:
"""Tests the structural XML diff logic in verify_success."""
def _make_memory(self):
fake_db = FakeUIMemoryDB()
return ActionMemory(ui_memory=fake_db), fake_db
def test_toggle_massive_shift_is_navigation_error(self):
"""
If a 'like' click causes >1000 char XML diff, the bot
accidentally navigated away. Must return False.
"""
memory, _ = self._make_memory()
pre_xml = "<node>" + "x" * 500 + "</node>"
post_xml = "<completely_different>" + "y" * 2000 + "</completely_different>"
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
assert result is False
def test_toggle_small_diff_is_success(self):
"""A small structural change for a toggle intent = success."""
memory, _ = self._make_memory()
# Track a click so the semantic gate has context
node = _make_node(content_desc="Like", resource_id="like_button")
memory.track_click("like", node)
pre_xml = '<node text="Like" />'
post_xml = '<node text="Liked" />' # Small change
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
assert result is True
def test_toggle_zero_diff_is_none(self):
"""Zero XML change for a toggle = inconclusive (not confirmed)."""
memory, _ = self._make_memory()
same_xml = '<node text="Like" />'
result = memory.verify_success("like", pre_click_xml=same_xml, post_click_xml=same_xml)
# Zero diff, no markers → None (inconclusive) not True
assert result is None or result is False
def test_follow_success_with_requested_marker(self):
"""
Private accounts show 'Requested' instead of 'Following'.
Must still count as success.
"""
memory, _ = self._make_memory()
post_xml = '<node text="Requested" />'
result = memory.verify_success("follow", pre_click_xml="<node/>", post_click_xml=post_xml)
assert result is True
def test_follow_success_german_locale(self):
"""German locale uses 'Abonniert' or 'Angefragt'."""
memory, _ = self._make_memory()
post_xml = '<node text="Abonniert" />'
result = memory.verify_success("follow", pre_click_xml="<node/>", post_click_xml=post_xml)
assert result is True
def test_view_post_clips_viewer_marker(self):
"""Opening a Reel shows clips_viewer_view_pager — must be detected."""
memory, _ = self._make_memory()
post_xml = '<node resource-id="com.instagram.android:id/clips_viewer_view_pager" />'
result = memory.verify_success("view a post", pre_click_xml="<node/>", post_click_xml=post_xml)
assert result is True
def test_view_post_still_on_grid_is_inconclusive(self):
"""
If after 'view a post' the XML still shows explore_action_bar
WITHOUT post detail markers, navigation failed.
"""
memory, _ = self._make_memory()
post_xml = '<node resource-id="com.instagram.android:id/explore_action_bar" />'
result = memory.verify_success("grid item", pre_click_xml="<node/>", post_click_xml=post_xml)
assert result is None # Inconclusive
def test_semantic_gate_blocks_wrong_toggle_element(self):
"""
If a 'like' click was tracked on a caption (not a like button),
verify_success must reject it via the semantic gate.
"""
memory, _ = self._make_memory()
node = _make_node(
text="Beautiful sunset photo!",
resource_id="caption_text",
content_desc="",
)
memory.track_click("like", node)
pre_xml = '<node text="Like" />'
post_xml = '<node text="Liked" />'
result = memory.verify_success("like", pre_click_xml=pre_xml, post_click_xml=post_xml)
assert result is False