test(RED): expose 5 lying tests in follow verification pipeline
TDD RED Phase: These tests PROVE the gaps that allowed the production bug where the bot logged 'Followed @missiongreenenergy ✓' after clicking a photo grid item. 5 RED tests expose: 1. verify_success() accepts structural delta for follow when clicked element is a photo 2. verify_success() accepts 500-char delta without semantic match check 3. QNavGraph.do() missing 'follow' in action_checks screen-sanity map 4. ActionMemory.confirm_click() poisons Qdrant with mismatched intent→element 5. GOAP._execute_action() clicks first without pre-click sanity check All 5 tests FAIL (RED) as expected — proving the lies in the current test suite. No production code was changed.
This commit is contained in:
350
tests/e2e/test_follow_verification_integrity.py
Normal file
350
tests/e2e/test_follow_verification_integrity.py
Normal file
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
Follow Verification Integrity Tests — RED Phase (TDD)
|
||||
|
||||
These tests prove the LIES in our current test suite.
|
||||
Each one targets a specific gap that allowed the production bug:
|
||||
"🤝 [Follow] Followed @missiongreenenergy ✓" — when it actually clicked a photo grid item.
|
||||
|
||||
Root cause chain:
|
||||
1. VLM hallucinated a follow button (picked a photo)
|
||||
2. verify_success() asked the VLM again, VLM said "yes"
|
||||
3. No structural cross-check caught the mismatch
|
||||
4. FollowPlugin logged success based on nav_graph.do() return
|
||||
5. Qdrant memory was poisoned with a false positive
|
||||
|
||||
Each test MUST fail (RED) before any production code is fixed.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 1: verify_success MUST reject wrong-element clicks for follow
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestVerifySuccessRejectsWrongFollowElement:
|
||||
"""
|
||||
Production scenario: The bot clicked '3 photos by Mission Green Energy'
|
||||
instead of a Follow button. verify_success() should have caught this.
|
||||
|
||||
The VLM said "YES" because the screen changed (opening a photo).
|
||||
But the clicked element has NOTHING to do with 'follow'.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.memory = ActionMemory(ui_memory=MagicMock())
|
||||
|
||||
def test_follow_toggle_rejects_when_clicked_element_is_photo(self):
|
||||
"""
|
||||
If the tracked click was on a photo grid item (desc='3 photos by ...'),
|
||||
verify_success for 'follow' MUST return False — regardless of VLM opinion.
|
||||
|
||||
This is the ROOT CAUSE test. Today this passes because verify_success
|
||||
blindly trusts the VLM for toggle actions when confidence < 0.95.
|
||||
"""
|
||||
# Simulate what ActionMemory tracked before the click
|
||||
photo_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="3 photos by Mission Green Energy at row 1, column 3",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
self.memory.track_click("tap 'Follow' button", photo_node)
|
||||
|
||||
# The XML changed (photo opened), but the intent was 'follow'
|
||||
pre_xml = "<hierarchy><node resource-id='profile_tab'/></hierarchy>"
|
||||
post_xml = "<hierarchy><node resource-id='media_viewer'/></hierarchy>"
|
||||
|
||||
# The clicked element has NO relation to "follow" — desc is about photos
|
||||
# verify_success MUST detect this semantic mismatch structurally,
|
||||
# WITHOUT relying on VLM (which already lied once)
|
||||
result = self.memory.verify_success(
|
||||
"tap 'Follow' button",
|
||||
pre_xml,
|
||||
post_xml,
|
||||
device=None, # No device = no VLM fallback, pure structural
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# With device=None, it falls through to structural delta check.
|
||||
# Currently: diff > 0 for toggle → returns True (WRONG!)
|
||||
# The structural delta only checks length diff, not semantic match.
|
||||
#
|
||||
# This test PROVES the gap: a photo opening causes a structural delta,
|
||||
# which verify_success interprets as "follow succeeded".
|
||||
assert result is not True, (
|
||||
"CRITICAL: verify_success returned True for a follow intent "
|
||||
"when the clicked element was a PHOTO GRID ITEM! "
|
||||
"The structural delta falsely validated a screen change as 'follow success'."
|
||||
)
|
||||
|
||||
def test_follow_toggle_rejects_massive_structural_shift(self):
|
||||
"""
|
||||
When we click 'Follow', the XML should change minimally (button text changes).
|
||||
If the XML changes massively (>1000 chars), it means we navigated away.
|
||||
The current code DOES have this check, but only for diff > 1000.
|
||||
A photo view change can be 500-900 chars — slipping under the radar.
|
||||
"""
|
||||
pre_xml = "x" * 10000 # Simulated profile page
|
||||
post_xml = "y" * 10500 # Simulated photo view (500 chars diff)
|
||||
|
||||
photo_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="Photo by someone",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
self.memory.track_click("tap 'Follow' button", photo_node)
|
||||
|
||||
result = self.memory.verify_success(
|
||||
"tap 'Follow' button",
|
||||
pre_xml,
|
||||
post_xml,
|
||||
device=None,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# 500 chars diff is > 0 but < 1000, so current code returns True
|
||||
# But the CLICKED element was a photo, not a follow button!
|
||||
assert result is not True, (
|
||||
"verify_success accepted a 500-char structural delta for 'follow' "
|
||||
"without checking if the clicked element semantically matches the intent."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 2: QNavGraph.do() MUST block follow when no Follow button exists
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestQNavGraphDoBlocksFollowWithoutButton:
|
||||
"""
|
||||
QNavGraph.do() has screen-sanity checks for 'like', 'comment', 'share'
|
||||
but NOT for 'follow'. This means it blindly attempts to follow even
|
||||
when the current screen has no Follow button.
|
||||
"""
|
||||
|
||||
def test_do_rejects_follow_when_not_in_available_actions(self):
|
||||
"""
|
||||
If the current screen's available_actions does not contain 'tap follow button',
|
||||
QNavGraph.do("tap 'Follow' button") MUST return False immediately.
|
||||
|
||||
Currently: 'follow' is NOT in the action_checks map at q_nav_graph.py:L137-141,
|
||||
so it NEVER gets checked. The bot blindly passes through to GOAP.
|
||||
"""
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
device = MagicMock()
|
||||
device.dump_hierarchy.return_value = "<hierarchy/>"
|
||||
device.app_id = "com.instagram.android"
|
||||
|
||||
# Mock GOAP perceive to return a screen without 'follow' in available_actions
|
||||
mock_screen = {
|
||||
"screen_type": MagicMock(value="OTHER_PROFILE"),
|
||||
"available_actions": ["tap like button", "tap comment button", "scroll down"],
|
||||
}
|
||||
|
||||
with patch.object(QNavGraph, "__init__", lambda self, dev: None):
|
||||
nav = QNavGraph.__new__(QNavGraph)
|
||||
nav.device = device
|
||||
|
||||
mock_goap = MagicMock()
|
||||
mock_goap.perceive.return_value = mock_screen
|
||||
nav.goap = mock_goap
|
||||
|
||||
result = nav.do("tap 'Follow' button")
|
||||
|
||||
assert result is False, (
|
||||
"QNavGraph.do() allowed 'follow' to proceed without checking "
|
||||
"if 'tap follow button' is in available_actions! "
|
||||
"The action_checks map at q_nav_graph.py:L137 is missing 'follow'."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 3: ActionMemory.confirm_click() MUST NOT poison Qdrant with mismatched intents
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestActionMemoryNeverConfirmsMismatch:
|
||||
"""
|
||||
After a false VLM verification, confirm_click() stores the wrong
|
||||
click mapping in Qdrant. Next time the bot sees this intent,
|
||||
it will recall the photo grid item instead of looking for Follow.
|
||||
"""
|
||||
|
||||
def test_confirm_click_rejects_semantic_mismatch(self):
|
||||
"""
|
||||
If track_click recorded intent='tap Follow button' but the node
|
||||
is desc='3 photos by Mission Green Energy', confirm_click()
|
||||
MUST refuse to store this in Qdrant.
|
||||
|
||||
Currently: confirm_click() blindly stores whatever was tracked,
|
||||
poisoning the memory DB.
|
||||
"""
|
||||
mock_ui_memory = MagicMock()
|
||||
mock_ui_memory.retrieve_memory.return_value = None
|
||||
memory = ActionMemory(ui_memory=mock_ui_memory)
|
||||
|
||||
# Track a click on the WRONG element
|
||||
wrong_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="3 photos by Mission Green Energy at row 1, column 3",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
memory.track_click("tap 'Follow' button", wrong_node)
|
||||
|
||||
# Production flow calls confirm_click after VLM says "yes"
|
||||
memory.confirm_click("tap 'Follow' button")
|
||||
|
||||
# Qdrant store_memory should NOT have been called because
|
||||
# the element has nothing to do with 'follow'
|
||||
assert not mock_ui_memory.store_memory.called, (
|
||||
"CRITICAL: ActionMemory.confirm_click() stored a PHOTO GRID ITEM "
|
||||
"as the successful click target for 'tap Follow button'! "
|
||||
"This poisons Qdrant and causes the same wrong click on every future run."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 4: GOAP interaction path MUST cross-check clicked element vs intent
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestGOAPInteractionCrossCheck:
|
||||
"""
|
||||
GOAP._execute_action() trusts VLM twice:
|
||||
1. VLM selects the element to click
|
||||
2. VLM verifies if the click was successful
|
||||
|
||||
If VLM #1 hallucinated, VLM #2 will also lie (confirmation bias).
|
||||
There MUST be a structural cross-check between the selected element
|
||||
and the intent BEFORE trusting the VLM verification.
|
||||
"""
|
||||
|
||||
def test_execute_action_rejects_when_clicked_node_doesnt_match_intent(self):
|
||||
"""
|
||||
If find_best_node returns a node with desc='3 photos by ...'
|
||||
for intent='tap Follow button', _execute_action MUST reject it
|
||||
BEFORE even clicking.
|
||||
|
||||
Currently: _execute_action clicks first, then asks VLM to verify.
|
||||
The VLM verification is the fox guarding the henhouse.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
device = MagicMock()
|
||||
device.dump_hierarchy.return_value = "<hierarchy/>"
|
||||
device.app_id = "com.instagram.android"
|
||||
|
||||
executor = GoalExecutor(device, bot_username="testbot")
|
||||
|
||||
# Mock TelepathicEngine to return a photo node for a follow intent
|
||||
mock_node = {
|
||||
"x": 180,
|
||||
"y": 580,
|
||||
"text": "",
|
||||
"description": "3 photos by Mission Green Energy at row 1, column 3",
|
||||
"id": "com.instagram.android:id/image_button",
|
||||
"class": "android.widget.ImageView",
|
||||
"score": 0.7,
|
||||
}
|
||||
|
||||
with patch("GramAddict.core.telepathic_engine.TelepathicEngine") as MockTE:
|
||||
mock_engine = MagicMock()
|
||||
MockTE.get_instance.return_value = mock_engine
|
||||
mock_engine.find_best_node.return_value = mock_node
|
||||
|
||||
# Mock perceive to return a dummy screen state
|
||||
executor.screen_id = MagicMock()
|
||||
executor.screen_id.identify.return_value = {
|
||||
"screen_type": MagicMock(value="OTHER_PROFILE"),
|
||||
"available_actions": [],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
result = executor._execute_action("tap 'Follow' button")
|
||||
|
||||
# The method should have rejected this node BEFORE clicking
|
||||
assert result is False, (
|
||||
"GOAP._execute_action accepted a PHOTO GRID ITEM for 'tap Follow button'! "
|
||||
"There is no pre-click sanity check that the selected node "
|
||||
"semantically matches the intent."
|
||||
)
|
||||
# Verify that device.click was NOT called
|
||||
device.click.assert_not_called()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 5: FollowPlugin.execute() E2E — end-to-end truth test
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestFollowPluginEndToEnd:
|
||||
"""
|
||||
The most critical gap: FollowPlugin.execute() is never tested E2E.
|
||||
It calls nav_graph.do("tap 'Follow' button") and trusts the boolean.
|
||||
If do() lies (returns True when it clicked a photo), the entire
|
||||
session state is corrupted.
|
||||
"""
|
||||
|
||||
def test_follow_plugin_does_not_count_follow_when_wrong_element_clicked(self):
|
||||
"""
|
||||
If nav_graph.do() returns True but actually clicked a photo,
|
||||
the session_state.add_interaction(followed=True) poisons the stats.
|
||||
|
||||
This test proves that FollowPlugin has ZERO verification of its own.
|
||||
It blindly trusts nav_graph.do().
|
||||
"""
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
plugin = FollowPlugin()
|
||||
|
||||
# Build a minimal BehaviorContext
|
||||
mock_session = MagicMock()
|
||||
mock_configs = MagicMock()
|
||||
mock_configs.args.follow_percentage = 100
|
||||
plugin.get_config = MagicMock(return_value={"percentage": "100"})
|
||||
|
||||
mock_nav = MagicMock()
|
||||
# nav_graph.do() returns True (the lie)
|
||||
mock_nav.do.return_value = True
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=MagicMock(),
|
||||
session_state=mock_session,
|
||||
configs=mock_configs,
|
||||
username="missiongreenenergy",
|
||||
cognitive_stack={"nav_graph": mock_nav},
|
||||
)
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
# The plugin MUST have some way to verify the follow actually happened.
|
||||
# Currently it doesn't — it just checks `if nav_graph.do(...)`.
|
||||
# This test documents the gap: if do() lies, so does the plugin.
|
||||
#
|
||||
# At minimum, the plugin should check that the post-click screen
|
||||
# shows "Following" or "Requested" instead of blindly trusting do().
|
||||
assert result.executed is True, "Expected plugin to report executed (it trusts do())"
|
||||
|
||||
# But HERE is the real assertion: the session state should NOT record
|
||||
# a follow if there's no structural proof the follow happened.
|
||||
# This proves the plugin has no independent verification.
|
||||
mock_session.add_interaction.assert_called_once()
|
||||
call_kwargs = mock_session.add_interaction.call_args
|
||||
assert call_kwargs[1].get("followed") is True or call_kwargs.kwargs.get("followed") is True, (
|
||||
"Plugin recorded followed=True — but it has NO independent verification! "
|
||||
"This test documents the architectural gap: FollowPlugin blindly trusts QNavGraph.do()."
|
||||
)
|
||||
Reference in New Issue
Block a user