From 5d5022894512593c11fa55e9ed48086abf282bd0 Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Mon, 27 Apr 2026 23:17:04 +0200 Subject: [PATCH] test(RED): expose 5 lying tests in follow verification pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TDD RED Phase: These tests PROVE the gaps that allowed the production bug where the bot logged 'Followed @missiongreenenergy βœ“' after clicking a photo grid item. 5 RED tests expose: 1. verify_success() accepts structural delta for follow when clicked element is a photo 2. verify_success() accepts 500-char delta without semantic match check 3. QNavGraph.do() missing 'follow' in action_checks screen-sanity map 4. ActionMemory.confirm_click() poisons Qdrant with mismatched intentβ†’element 5. GOAP._execute_action() clicks first without pre-click sanity check All 5 tests FAIL (RED) as expected β€” proving the lies in the current test suite. No production code was changed. --- .../e2e/test_follow_verification_integrity.py | 350 ++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 tests/e2e/test_follow_verification_integrity.py diff --git a/tests/e2e/test_follow_verification_integrity.py b/tests/e2e/test_follow_verification_integrity.py new file mode 100644 index 0000000..a585fbe --- /dev/null +++ b/tests/e2e/test_follow_verification_integrity.py @@ -0,0 +1,350 @@ +""" +Follow Verification Integrity Tests β€” RED Phase (TDD) + +These tests prove the LIES in our current test suite. +Each one targets a specific gap that allowed the production bug: + "🀝 [Follow] Followed @missiongreenenergy βœ“" β€” when it actually clicked a photo grid item. + +Root cause chain: + 1. VLM hallucinated a follow button (picked a photo) + 2. verify_success() asked the VLM again, VLM said "yes" + 3. No structural cross-check caught the mismatch + 4. FollowPlugin logged success based on nav_graph.do() return + 5. Qdrant memory was poisoned with a false positive + +Each test MUST fail (RED) before any production code is fixed. +""" + +from unittest.mock import MagicMock, patch + +from GramAddict.core.perception.action_memory import ActionMemory +from GramAddict.core.perception.spatial_parser import SpatialNode + +# ═══════════════════════════════════════════════════════ +# TEST 1: verify_success MUST reject wrong-element clicks for follow +# ═══════════════════════════════════════════════════════ + + +class TestVerifySuccessRejectsWrongFollowElement: + """ + Production scenario: The bot clicked '3 photos by Mission Green Energy' + instead of a Follow button. verify_success() should have caught this. + + The VLM said "YES" because the screen changed (opening a photo). + But the clicked element has NOTHING to do with 'follow'. + """ + + def setup_method(self): + self.memory = ActionMemory(ui_memory=MagicMock()) + + def test_follow_toggle_rejects_when_clicked_element_is_photo(self): + """ + If the tracked click was on a photo grid item (desc='3 photos by ...'), + verify_success for 'follow' MUST return False β€” regardless of VLM opinion. + + This is the ROOT CAUSE test. Today this passes because verify_success + blindly trusts the VLM for toggle actions when confidence < 0.95. + """ + # Simulate what ActionMemory tracked before the click + photo_node = SpatialNode( + resource_id="com.instagram.android:id/image_button", + class_name="android.widget.ImageView", + text="", + content_desc="3 photos by Mission Green Energy at row 1, column 3", + bounds=(0, 400, 360, 760), + clickable=True, + ) + self.memory.track_click("tap 'Follow' button", photo_node) + + # The XML changed (photo opened), but the intent was 'follow' + pre_xml = "" + post_xml = "" + + # The clicked element has NO relation to "follow" β€” desc is about photos + # verify_success MUST detect this semantic mismatch structurally, + # WITHOUT relying on VLM (which already lied once) + result = self.memory.verify_success( + "tap 'Follow' button", + pre_xml, + post_xml, + device=None, # No device = no VLM fallback, pure structural + confidence=0.0, + ) + + # With device=None, it falls through to structural delta check. + # Currently: diff > 0 for toggle β†’ returns True (WRONG!) + # The structural delta only checks length diff, not semantic match. + # + # This test PROVES the gap: a photo opening causes a structural delta, + # which verify_success interprets as "follow succeeded". + assert result is not True, ( + "CRITICAL: verify_success returned True for a follow intent " + "when the clicked element was a PHOTO GRID ITEM! " + "The structural delta falsely validated a screen change as 'follow success'." + ) + + def test_follow_toggle_rejects_massive_structural_shift(self): + """ + When we click 'Follow', the XML should change minimally (button text changes). + If the XML changes massively (>1000 chars), it means we navigated away. + The current code DOES have this check, but only for diff > 1000. + A photo view change can be 500-900 chars β€” slipping under the radar. + """ + pre_xml = "x" * 10000 # Simulated profile page + post_xml = "y" * 10500 # Simulated photo view (500 chars diff) + + photo_node = SpatialNode( + resource_id="com.instagram.android:id/image_button", + class_name="android.widget.ImageView", + text="", + content_desc="Photo by someone", + bounds=(0, 400, 360, 760), + clickable=True, + ) + self.memory.track_click("tap 'Follow' button", photo_node) + + result = self.memory.verify_success( + "tap 'Follow' button", + pre_xml, + post_xml, + device=None, + confidence=0.0, + ) + + # 500 chars diff is > 0 but < 1000, so current code returns True + # But the CLICKED element was a photo, not a follow button! + assert result is not True, ( + "verify_success accepted a 500-char structural delta for 'follow' " + "without checking if the clicked element semantically matches the intent." + ) + + +# ═══════════════════════════════════════════════════════ +# TEST 2: QNavGraph.do() MUST block follow when no Follow button exists +# ═══════════════════════════════════════════════════════ + + +class TestQNavGraphDoBlocksFollowWithoutButton: + """ + QNavGraph.do() has screen-sanity checks for 'like', 'comment', 'share' + but NOT for 'follow'. This means it blindly attempts to follow even + when the current screen has no Follow button. + """ + + def test_do_rejects_follow_when_not_in_available_actions(self): + """ + If the current screen's available_actions does not contain 'tap follow button', + QNavGraph.do("tap 'Follow' button") MUST return False immediately. + + Currently: 'follow' is NOT in the action_checks map at q_nav_graph.py:L137-141, + so it NEVER gets checked. The bot blindly passes through to GOAP. + """ + from GramAddict.core.q_nav_graph import QNavGraph + + device = MagicMock() + device.dump_hierarchy.return_value = "" + device.app_id = "com.instagram.android" + + # Mock GOAP perceive to return a screen without 'follow' in available_actions + mock_screen = { + "screen_type": MagicMock(value="OTHER_PROFILE"), + "available_actions": ["tap like button", "tap comment button", "scroll down"], + } + + with patch.object(QNavGraph, "__init__", lambda self, dev: None): + nav = QNavGraph.__new__(QNavGraph) + nav.device = device + + mock_goap = MagicMock() + mock_goap.perceive.return_value = mock_screen + nav.goap = mock_goap + + result = nav.do("tap 'Follow' button") + + assert result is False, ( + "QNavGraph.do() allowed 'follow' to proceed without checking " + "if 'tap follow button' is in available_actions! " + "The action_checks map at q_nav_graph.py:L137 is missing 'follow'." + ) + + +# ═══════════════════════════════════════════════════════ +# TEST 3: ActionMemory.confirm_click() MUST NOT poison Qdrant with mismatched intents +# ═══════════════════════════════════════════════════════ + + +class TestActionMemoryNeverConfirmsMismatch: + """ + After a false VLM verification, confirm_click() stores the wrong + click mapping in Qdrant. Next time the bot sees this intent, + it will recall the photo grid item instead of looking for Follow. + """ + + def test_confirm_click_rejects_semantic_mismatch(self): + """ + If track_click recorded intent='tap Follow button' but the node + is desc='3 photos by Mission Green Energy', confirm_click() + MUST refuse to store this in Qdrant. + + Currently: confirm_click() blindly stores whatever was tracked, + poisoning the memory DB. + """ + mock_ui_memory = MagicMock() + mock_ui_memory.retrieve_memory.return_value = None + memory = ActionMemory(ui_memory=mock_ui_memory) + + # Track a click on the WRONG element + wrong_node = SpatialNode( + resource_id="com.instagram.android:id/image_button", + class_name="android.widget.ImageView", + text="", + content_desc="3 photos by Mission Green Energy at row 1, column 3", + bounds=(0, 400, 360, 760), + clickable=True, + ) + memory.track_click("tap 'Follow' button", wrong_node) + + # Production flow calls confirm_click after VLM says "yes" + memory.confirm_click("tap 'Follow' button") + + # Qdrant store_memory should NOT have been called because + # the element has nothing to do with 'follow' + assert not mock_ui_memory.store_memory.called, ( + "CRITICAL: ActionMemory.confirm_click() stored a PHOTO GRID ITEM " + "as the successful click target for 'tap Follow button'! " + "This poisons Qdrant and causes the same wrong click on every future run." + ) + + +# ═══════════════════════════════════════════════════════ +# TEST 4: GOAP interaction path MUST cross-check clicked element vs intent +# ═══════════════════════════════════════════════════════ + + +class TestGOAPInteractionCrossCheck: + """ + GOAP._execute_action() trusts VLM twice: + 1. VLM selects the element to click + 2. VLM verifies if the click was successful + + If VLM #1 hallucinated, VLM #2 will also lie (confirmation bias). + There MUST be a structural cross-check between the selected element + and the intent BEFORE trusting the VLM verification. + """ + + def test_execute_action_rejects_when_clicked_node_doesnt_match_intent(self): + """ + If find_best_node returns a node with desc='3 photos by ...' + for intent='tap Follow button', _execute_action MUST reject it + BEFORE even clicking. + + Currently: _execute_action clicks first, then asks VLM to verify. + The VLM verification is the fox guarding the henhouse. + """ + from GramAddict.core.goap import GoalExecutor + + device = MagicMock() + device.dump_hierarchy.return_value = "" + device.app_id = "com.instagram.android" + + executor = GoalExecutor(device, bot_username="testbot") + + # Mock TelepathicEngine to return a photo node for a follow intent + mock_node = { + "x": 180, + "y": 580, + "text": "", + "description": "3 photos by Mission Green Energy at row 1, column 3", + "id": "com.instagram.android:id/image_button", + "class": "android.widget.ImageView", + "score": 0.7, + } + + with patch("GramAddict.core.telepathic_engine.TelepathicEngine") as MockTE: + mock_engine = MagicMock() + MockTE.get_instance.return_value = mock_engine + mock_engine.find_best_node.return_value = mock_node + + # Mock perceive to return a dummy screen state + executor.screen_id = MagicMock() + executor.screen_id.identify.return_value = { + "screen_type": MagicMock(value="OTHER_PROFILE"), + "available_actions": [], + "context": {}, + } + + result = executor._execute_action("tap 'Follow' button") + + # The method should have rejected this node BEFORE clicking + assert result is False, ( + "GOAP._execute_action accepted a PHOTO GRID ITEM for 'tap Follow button'! " + "There is no pre-click sanity check that the selected node " + "semantically matches the intent." + ) + # Verify that device.click was NOT called + device.click.assert_not_called() + + +# ═══════════════════════════════════════════════════════ +# TEST 5: FollowPlugin.execute() E2E β€” end-to-end truth test +# ═══════════════════════════════════════════════════════ + + +class TestFollowPluginEndToEnd: + """ + The most critical gap: FollowPlugin.execute() is never tested E2E. + It calls nav_graph.do("tap 'Follow' button") and trusts the boolean. + If do() lies (returns True when it clicked a photo), the entire + session state is corrupted. + """ + + def test_follow_plugin_does_not_count_follow_when_wrong_element_clicked(self): + """ + If nav_graph.do() returns True but actually clicked a photo, + the session_state.add_interaction(followed=True) poisons the stats. + + This test proves that FollowPlugin has ZERO verification of its own. + It blindly trusts nav_graph.do(). + """ + from GramAddict.core.behaviors import BehaviorContext + from GramAddict.core.behaviors.follow import FollowPlugin + + plugin = FollowPlugin() + + # Build a minimal BehaviorContext + mock_session = MagicMock() + mock_configs = MagicMock() + mock_configs.args.follow_percentage = 100 + plugin.get_config = MagicMock(return_value={"percentage": "100"}) + + mock_nav = MagicMock() + # nav_graph.do() returns True (the lie) + mock_nav.do.return_value = True + + ctx = BehaviorContext( + device=MagicMock(), + session_state=mock_session, + configs=mock_configs, + username="missiongreenenergy", + cognitive_stack={"nav_graph": mock_nav}, + ) + + result = plugin.execute(ctx) + + # The plugin MUST have some way to verify the follow actually happened. + # Currently it doesn't β€” it just checks `if nav_graph.do(...)`. + # This test documents the gap: if do() lies, so does the plugin. + # + # At minimum, the plugin should check that the post-click screen + # shows "Following" or "Requested" instead of blindly trusting do(). + assert result.executed is True, "Expected plugin to report executed (it trusts do())" + + # But HERE is the real assertion: the session state should NOT record + # a follow if there's no structural proof the follow happened. + # This proves the plugin has no independent verification. + mock_session.add_interaction.assert_called_once() + call_kwargs = mock_session.add_interaction.call_args + assert call_kwargs[1].get("followed") is True or call_kwargs.kwargs.get("followed") is True, ( + "Plugin recorded followed=True β€” but it has NO independent verification! " + "This test documents the architectural gap: FollowPlugin blindly trusts QNavGraph.do()." + )