diff --git a/tests/e2e/test_system_llm_perception.py b/tests/e2e/test_system_llm_perception.py
new file mode 100644
index 0000000..babe13d
--- /dev/null
+++ b/tests/e2e/test_system_llm_perception.py
@@ -0,0 +1,517 @@
+"""
+E2E: LLM Perception Pipeline Integrity
+========================================
+Tests the LLM/VLM integration layer WITHOUT calling a real LLM.
+
+These tests validate the deterministic processing logic AROUND the LLM:
+- Response parsing (what happens with malformed JSON, empty strings, timeouts)
+- Structural guards (do they block VLM hallucinations?)
+- Box index extraction from diverse response formats
+- ActionMemory confidence tracking (boost/penalty math)
+- Semantic match validation (prevents memory poisoning)
+
+WHY THIS MATTERS (from 2026-05-02 production log):
+ VLM was asked to "tap back button" but selected box [0] (a Reel thumbnail)
+ instead of box [22] (desc='Back'). The structural guards SHOULD have
+ prevented this, but were never tested.
+"""
+
+import json
+
+import pytest
+
+from GramAddict.core.perception.action_memory import (
+ ActionMemory,
+ _intent_matches_node,
+ _parse_yes_no,
+)
+from GramAddict.core.perception.intent_resolver import IntentResolver
+from GramAddict.core.perception.spatial_parser import SpatialNode
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# Fake UIMemoryDB — replaces MagicMock to satisfy mock ban
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class FakeUIMemoryDB:
+ """
+ Real fake implementation of UIMemoryDB that tracks method calls
+ without using unittest.mock. Satisfies the project's strict mock ban.
+ """
+
+ def __init__(self):
+ self.store_memory_calls = []
+ self.boost_confidence_calls = []
+ self.decay_confidence_calls = []
+ self.retrieve_memory_calls = []
+
+ def retrieve_memory(self, intent, xml_context):
+ self.retrieve_memory_calls.append((intent, xml_context))
+ return None
+
+ def store_memory(self, intent, xml_context, node_dict):
+ self.store_memory_calls.append((intent, xml_context, node_dict))
+
+ def boost_confidence(self, intent, xml_context):
+ self.boost_confidence_calls.append((intent, xml_context))
+
+ def decay_confidence(self, intent, xml_context):
+ self.decay_confidence_calls.append((intent, xml_context))
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# Helpers
+# ═══════════════════════════════════════════════════════════════════════
+
+
+def _make_node(
+ resource_id="",
+ content_desc="",
+ text="",
+ clickable=True,
+ bounds=(0, 0, 100, 100),
+ center_y=None,
+):
+ """Create a SpatialNode for testing.
+
+ If center_y is specified, adjusts bounds to produce that center_y value.
+ """
+ if center_y is not None:
+ # Recalculate bounds to produce the desired center_y
+ # Keep x bounds as-is, adjust y bounds symmetrically around center_y
+ half_height = (bounds[3] - bounds[1]) // 2
+ bounds = (bounds[0], center_y - half_height, bounds[2], center_y + half_height)
+
+ return SpatialNode(
+ bounds=bounds,
+ node_id=f"node_{resource_id or text or content_desc}",
+ text=text,
+ content_desc=content_desc,
+ resource_id=resource_id,
+ clickable=clickable,
+ )
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# CONTRACT 1: VLM Response Parsing — _parse_yes_no
+# ═══════════════════════════════════════════════════════════════════════
+#
+# This function parses every VLM verification response. If it misparses,
+# the bot either accepts failed clicks or rejects successful ones.
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class TestVLMResponseParsing:
+ """Validates _parse_yes_no handles every response format the VLM produces."""
+
+ # Known VLM response formats from production logs
+ @pytest.mark.parametrize(
+ "response,expected",
+ [
+ # Clean responses
+ ("YES", True),
+ ("NO", False),
+ ("yes", True),
+ ("no", False),
+ # JSON responses (seen in production)
+ ('{"YES": 1}', True),
+ ('{"NO": 1}', False),
+ ('{"answer": "YES"}', True),
+ ('{"answer": "NO"}', False),
+ ('{ "YES": 1 }', True),
+ # Responses with explanations (VLM ignores instructions)
+ ("YES, the button was tapped successfully.", True),
+ ("NO, the screen did not change.", False),
+ ("Yes - the action succeeded.", True),
+ ("No - navigation failed.", False),
+ # Edge cases that must NOT match
+ ("now loading", None), # "now" starts with "no"
+ ("not applicable", None), # "not" starts with "no"
+ ("nothing changed", None), # contains "no" but not standalone
+ # Empty/garbage
+ ("", None),
+ (" ", None),
+ # Free-form that happened in production
+ (
+ '{ "intent": "tap back button", "element_tapped": "some reel" }',
+ None,
+ ),
+ # Ambiguous — starts with YES, so early-exit returns True
+ # NOTE: This is current behavior. If you want None here, fix _parse_yes_no.
+ ("YES and NO are both possible", True),
+ ],
+ ids=[
+ "clean_YES",
+ "clean_NO",
+ "lowercase_yes",
+ "lowercase_no",
+ "json_YES_key",
+ "json_NO_key",
+ "json_answer_YES",
+ "json_answer_NO",
+ "json_spaced_YES",
+ "yes_with_explanation",
+ "no_with_explanation",
+ "yes_dash",
+ "no_dash",
+ "now_loading",
+ "not_applicable",
+ "nothing_substring",
+ "empty",
+ "whitespace",
+ "freeform_json",
+ "ambiguous_both",
+ ],
+ )
+ def test_parse_yes_no(self, response, expected):
+ result = _parse_yes_no(response)
+ assert result is expected, (
+ f"_parse_yes_no('{response}') returned {result}, expected {expected}"
+ )
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# CONTRACT 2: VLM Box Index Extraction
+# ═══════════════════════════════════════════════════════════════════════
+#
+# The VLM returns {"box": N} but often varies the format.
+# The parsing logic in _visual_discovery must handle all variants.
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class TestBoxIndexParsing:
+ """Validates box index extraction from diverse VLM response formats."""
+
+ @pytest.mark.parametrize(
+ "vlm_response,expected_box",
+ [
+ ('{"box": 6}', 6),
+ ('{"box": 0}', 0),
+ ('{"box": null}', None),
+ ('{"selected_index": 3}', 3),
+ ('{"box_index": 12}', 12),
+ ('{"index": 5}', 5),
+ # Edge cases
+ ('{"box": 999}', 999), # Out of range — must not crash
+ ],
+ ids=["box_6", "box_0", "box_null", "selected_index", "box_index", "index", "out_of_range"],
+ )
+ def test_box_index_extraction(self, vlm_response, expected_box):
+ """Verify the JSON parsing logic extracts box indices correctly."""
+ data = json.loads(vlm_response)
+ box_idx = data.get("box")
+ if box_idx is None:
+ box_idx = data.get("selected_index")
+ if box_idx is None:
+ box_idx = data.get("box_index")
+ if box_idx is None:
+ box_idx = data.get("index")
+
+ assert box_idx == expected_box
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# CONTRACT 3: Structural Guards — Preventing VLM Hallucinations
+# ═══════════════════════════════════════════════════════════════════════
+#
+# These guards are the LAST LINE OF DEFENSE before a hallucinated click
+# hits the device. If they fail, the bot clicks wrong UI elements.
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class TestStructuralGuards:
+ """Tests the IntentResolver's structural guards that prevent VLM hallucinations."""
+
+ def setup_method(self):
+ self.resolver = IntentResolver()
+
+ # ── Navigation Conflict Guard ──
+
+ def test_tab_intent_excludes_back_button(self):
+ """
+ Production bug 2026-04-30: VLM picked 'Back' for 'tap profile tab'.
+ The guard must exclude Back/Close buttons for tab intents.
+ """
+ candidates = [
+ _make_node(resource_id="com.instagram.android:id/action_bar_button_back", content_desc="Back"),
+ _make_node(resource_id="com.instagram.android:id/profile_tab", content_desc="Profile", center_y=2350),
+ ]
+ filtered = self.resolver.filter_navigation_conflicts(candidates, "tap profile tab")
+ assert len(filtered) == 1
+ assert filtered[0].content_desc == "Profile"
+
+ def test_back_intent_keeps_back_button(self):
+ """Back intent must NOT filter out the Back button."""
+ candidates = [
+ _make_node(resource_id="com.instagram.android:id/action_bar_button_back", content_desc="Back"),
+ _make_node(resource_id="com.instagram.android:id/profile_tab", content_desc="Profile", center_y=2350),
+ ]
+ filtered = self.resolver.filter_navigation_conflicts(candidates, "tap back button")
+ # Back button must be preserved
+ back_nodes = [n for n in filtered if n.content_desc == "Back"]
+ assert len(back_nodes) == 1
+
+ def test_tab_intent_excludes_non_bottom_elements(self):
+ """
+ Tab elements are ALWAYS at the bottom (y > 85% of screen).
+ Elements higher up must be excluded for tab intents.
+ """
+ candidates = [
+ _make_node(content_desc="Search", center_y=200, bounds=(0, 150, 200, 250)), # Top search bar
+ _make_node(
+ resource_id="com.instagram.android:id/search_tab",
+ content_desc="Search and explore",
+ center_y=2350,
+ bounds=(216, 2300, 432, 2400),
+ ),
+ ]
+ filtered = self.resolver.filter_navigation_conflicts(
+ candidates, "tap explore tab", screen_height=2400
+ )
+ assert len(filtered) == 1
+ assert filtered[0].center_y == 2350
+
+ def test_author_intent_excludes_nav_tabs(self):
+ """
+ Production bug 2026-05-01: VLM picked 'Profile' tab for 'post author username'.
+ Author intents must never resolve to navigation tabs.
+ """
+ candidates = [
+ _make_node(
+ resource_id="com.instagram.android:id/row_feed_photo_profile_name",
+ text="photographer_jane",
+ clickable=True,
+ ),
+ _make_node(
+ resource_id="com.instagram.android:id/profile_tab",
+ content_desc="Profile",
+ center_y=2350,
+ ),
+ ]
+ filtered = self.resolver.filter_navigation_conflicts(
+ candidates, "tap post author username", screen_height=2400
+ )
+ author_nodes = [n for n in filtered if n.text == "photographer_jane"]
+ assert len(author_nodes) == 1
+
+ # ── Structural Fast-Paths ──
+
+ def test_fast_path_message_input(self):
+ """Message input must resolve structurally, never through VLM."""
+ candidates = [
+ _make_node(resource_id="com.instagram.android:id/composer_edittext", text="Message…"),
+ _make_node(content_desc="Like", resource_id="like_button"),
+ ]
+ result = self.resolver.resolve("message text box", candidates, device=None)
+ assert result is not None
+ assert "composer_edittext" in result.resource_id
+
+ def test_fast_path_send_button(self):
+ """Send button must resolve structurally."""
+ candidates = [
+ _make_node(resource_id="com.instagram.android:id/composer_send_button", content_desc="Send"),
+ _make_node(content_desc="Like"),
+ ]
+ result = self.resolver.resolve("send message button", candidates, device=None)
+ assert result is not None
+ assert "send" in result.resource_id.lower()
+
+ def test_fast_path_post_author(self):
+ """Post author username must resolve structurally."""
+ candidates = [
+ _make_node(
+ resource_id="com.instagram.android:id/row_feed_photo_profile_name",
+ text="photographer_jane",
+ ),
+ _make_node(content_desc="Profile", resource_id="profile_tab"),
+ ]
+ result = self.resolver.resolve("post author username", candidates, device=None)
+ assert result is not None
+ assert result.text == "photographer_jane"
+
+ def test_fast_path_comment_button(self):
+ """Comment button must resolve structurally."""
+ candidates = [
+ _make_node(resource_id="com.instagram.android:id/row_feed_button_comment", content_desc="Comment"),
+ _make_node(text="Nice photo!", resource_id="comment_text"),
+ ]
+ result = self.resolver.resolve("comment button", candidates, device=None)
+ assert result is not None
+ assert "comment" in result.resource_id.lower()
+
+ def test_abstract_goals_return_none(self):
+ """Abstract GOAP goals must never resolve to a node click."""
+ candidates = [
+ _make_node(text="anything", resource_id="whatever"),
+ ]
+ for goal in ["open profile", "open explore", "open following", "learn own profile"]:
+ result = self.resolver.resolve(goal, candidates, device=None)
+ assert result is None, f"Abstract goal '{goal}' should return None!"
+
+ def test_semantic_guard_quoted_target(self):
+ """Quoted targets must match exactly — VLM must not hallucinate."""
+ candidates = [
+ _make_node(text="New Message", resource_id="new_msg_btn"),
+ _make_node(text="Settings", resource_id="settings_btn"),
+ _make_node(text="Archive", resource_id="archive_btn"),
+ ]
+ result = self.resolver.resolve("tap 'New Message'", candidates, device=None)
+ assert result is not None
+ assert result.text == "New Message"
+
+ def test_semantic_guard_no_match_returns_none(self):
+ """If quoted target doesn't exist, return None (not hallucinate)."""
+ candidates = [
+ _make_node(text="Settings", resource_id="settings_btn"),
+ _make_node(text="Archive", resource_id="archive_btn"),
+ ]
+ result = self.resolver.resolve("tap 'Delete Account'", candidates, device=None)
+ assert result is None
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# CONTRACT 4: ActionMemory — Semantic Match Validation
+# ═══════════════════════════════════════════════════════════════════════
+#
+# Prevents memory poisoning: if the VLM clicks a Reel thumbnail for
+# "follow", the semantic guard must BLOCK confirmation into memory.
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class TestSemanticMatchGuard:
+ """Validates _intent_matches_node prevents memory poisoning."""
+
+ @pytest.mark.parametrize(
+ "intent,semantic_string,expected",
+ [
+ # Correct matches
+ ("follow", "text: 'Follow', desc: '', id: 'follow_button'", True),
+ ("like", "text: '', desc: 'Like', id: 'like_button'", True),
+ ("save", "text: 'Save', desc: 'Add to Saved', id: 'save_btn'", True),
+ # German locale
+ ("follow", "text: 'Abonnieren', desc: '', id: ''", True),
+ ("like", "text: '', desc: 'Gefällt mir', id: ''", True),
+ # POISONING ATTEMPTS — must be blocked
+ (
+ "follow",
+ "text: '', desc: 'Reel by trenny_m. View Count 3.143', id: 'preview_clip_thumbnail'",
+ False,
+ ),
+ (
+ "like",
+ "text: 'photographer_jane', desc: 'Photo by photographer_jane', id: 'feed_photo'",
+ False,
+ ),
+ ("save", "text: 'Nice photo!', desc: '', id: 'comment_text'", False),
+ # Non-toggle intents always pass
+ ("tap back button", "text: '', desc: 'Back', id: 'back_btn'", True),
+ ("open profile", "text: 'anything', desc: '', id: ''", True),
+ ],
+ ids=[
+ "follow_correct",
+ "like_correct",
+ "save_correct",
+ "follow_german",
+ "like_german",
+ "follow_reel_poison",
+ "like_photo_poison",
+ "save_comment_poison",
+ "back_non_toggle",
+ "abstract_non_toggle",
+ ],
+ )
+ def test_intent_matches_node(self, intent, semantic_string, expected):
+ result = _intent_matches_node(intent, semantic_string)
+ assert result is expected, (
+ f"_intent_matches_node('{intent}', '{semantic_string[:50]}...') "
+ f"returned {result}, expected {expected}"
+ )
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# CONTRACT 5: ActionMemory Lifecycle — Track → Verify → Confirm/Reject
+# ═══════════════════════════════════════════════════════════════════════
+#
+# This is the full lifecycle that runs for EVERY click in production.
+# If any step in this chain is broken, the bot either:
+# - Accepts hallucinated clicks (memory poisoning)
+# - Rejects correct clicks (performance death)
+# ═══════════════════════════════════════════════════════════════════════
+
+
+class TestActionMemoryLifecycle:
+ """Tests the full click tracking → confirmation/rejection lifecycle."""
+
+ def _make_memory(self):
+ """Create ActionMemory with a FakeUIMemoryDB."""
+ fake_db = FakeUIMemoryDB()
+ return ActionMemory(ui_memory=fake_db), fake_db
+
+ def test_confirm_correct_follow_stores_in_memory(self):
+ """A confirmed 'follow' click on a Follow button must be stored."""
+ memory, fake_db = self._make_memory()
+ node = _make_node(resource_id="follow_button", content_desc="Follow")
+ memory.track_click("follow", node)
+ memory.confirm_click("follow")
+ assert len(fake_db.store_memory_calls) == 1
+
+ def test_confirm_poisoned_follow_is_blocked(self):
+ """
+ Production bug: VLM clicked a Reel thumbnail for 'follow'.
+ The semantic guard must BLOCK this from being stored in memory.
+ """
+ memory, fake_db = self._make_memory()
+ node = _make_node(
+ resource_id="com.instagram.android:id/preview_clip_thumbnail",
+ content_desc="Reel by trenny_m. View Count 3.143",
+ )
+ memory.track_click("follow", node)
+ memory.confirm_click("follow")
+ # Must NOT have stored this poisoned memory
+ assert len(fake_db.store_memory_calls) == 0
+
+ def test_reject_click_triggers_decay(self):
+ """A rejected click must trigger confidence decay."""
+ memory, fake_db = self._make_memory()
+ node = _make_node(content_desc="Back")
+ memory.track_click("tap back button", node)
+ memory.reject_click("tap back button")
+ assert len(fake_db.decay_confidence_calls) == 1
+
+ def test_confirm_without_track_is_noop(self):
+ """Confirming without tracking must not crash or store."""
+ memory, fake_db = self._make_memory()
+ memory.confirm_click("follow") # No prior track_click
+ assert len(fake_db.store_memory_calls) == 0
+
+ def test_mismatched_intent_is_ignored(self):
+ """Confirming with a different intent than tracked must be ignored."""
+ memory, fake_db = self._make_memory()
+ node = _make_node(content_desc="Follow")
+ memory.track_click("follow", node)
+ memory.confirm_click("like") # Wrong intent
+ assert len(fake_db.store_memory_calls) == 0
+
+ def test_verify_follow_success_markers(self):
+ """
+ After a follow click, the post-click XML must contain
+ 'following' or 'requested' to count as success.
+ """
+ memory, _ = self._make_memory()
+
+ post_xml_success = ''
+ result = memory.verify_success(
+ "follow", pre_click_xml="", post_click_xml=post_xml_success
+ )
+ assert result is True
+
+ def test_verify_view_post_success(self):
+ """After tapping a grid item, post detail markers must appear."""
+ memory, _ = self._make_memory()
+
+ post_xml_success = ''
+ result = memory.verify_success(
+ "view a post", pre_click_xml="", post_click_xml=post_xml_success
+ )
+ assert result is True