Compare commits
2 Commits
7277f27fae
...
fix/lying-
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b9465a3bc | |||
| 5d50228945 |
@@ -321,6 +321,25 @@ class GoalExecutor:
|
||||
self._get_sae().ensure_clear_screen(max_attempts=3)
|
||||
return False
|
||||
|
||||
# ── Pre-Click Semantic Match Guard ──
|
||||
# For toggle intents (follow/like/save), verify the selected node
|
||||
# semantically matches the intent BEFORE clicking. This prevents
|
||||
# VLM hallucinations from clicking photo grid items when looking
|
||||
# for follow buttons.
|
||||
from GramAddict.core.perception.action_memory import _intent_matches_node
|
||||
|
||||
node_semantic = (
|
||||
f"text: '{best_node.get('text', '')}', "
|
||||
f"desc: '{best_node.get('description', '')}', "
|
||||
f"id: '{best_node.get('id', '')}'"
|
||||
)
|
||||
if not _intent_matches_node(action, node_semantic):
|
||||
logger.warning(
|
||||
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
|
||||
f"Node: {node_semantic}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Execute click
|
||||
self.device.click(obj=best_node)
|
||||
import random
|
||||
|
||||
@@ -5,6 +5,20 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Semantic Match Keywords — SSOT for intent → element validation
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
# Maps toggle-intent keywords to required element markers.
|
||||
# If the intent contains the key, the clicked element MUST
|
||||
# contain at least one of the corresponding markers in its
|
||||
# text, content_desc, or resource_id.
|
||||
TOGGLE_INTENT_MARKERS = {
|
||||
"follow": ["follow", "gefolgt", "abonnieren"],
|
||||
"like": ["like", "heart", "gefällt"],
|
||||
"save": ["save", "saved", "bookmark", "speichern"],
|
||||
}
|
||||
|
||||
|
||||
class ActionMemory:
|
||||
"""
|
||||
@@ -36,7 +50,11 @@ class ActionMemory:
|
||||
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
|
||||
|
||||
def confirm_click(self, intent: str = None):
|
||||
"""Positive Reinforcement: Confirms the last click was successful."""
|
||||
"""Positive Reinforcement: Confirms the last click was successful.
|
||||
|
||||
Guard: Refuses to store in Qdrant if the clicked element does not
|
||||
semantically match the intent. Prevents memory poisoning.
|
||||
"""
|
||||
ctx = self._last_click_context
|
||||
if not ctx:
|
||||
return
|
||||
@@ -44,6 +62,15 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
# ── Semantic Mismatch Guard ──
|
||||
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' — "
|
||||
f"clicked element does not match intent: {ctx['semantic_string']}"
|
||||
)
|
||||
self._last_click_context = None
|
||||
return
|
||||
|
||||
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
|
||||
|
||||
# Store or boost in Qdrant
|
||||
@@ -145,6 +172,18 @@ class ActionMemory:
|
||||
logger.error(f"Failed to query VLM for visual verification: {e}")
|
||||
# Fallthrough to structural delta if VLM crashes
|
||||
|
||||
# ── Pre-Structural Semantic Gate ──
|
||||
# Before trusting ANY structural delta, verify the clicked element
|
||||
# semantically matches the intent. Prevents photo-clicks from
|
||||
# being validated as follow/like successes.
|
||||
if is_toggle and self._last_click_context:
|
||||
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
|
||||
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
# Fallback to structural delta if no device, VLM fails, or high confidence bypass
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
|
||||
@@ -166,3 +205,30 @@ class ActionMemory:
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
"""Checks if the clicked element semantically matches the toggle intent.
|
||||
|
||||
For toggle intents (follow, like, save), the clicked element MUST contain
|
||||
at least one of the required keywords in its text/desc/id. This prevents
|
||||
photo grid items, captions, and other unrelated elements from being
|
||||
falsely confirmed as successful interactions.
|
||||
|
||||
For non-toggle intents, returns True (no restriction).
|
||||
"""
|
||||
intent_lower = intent.lower()
|
||||
semantic_lower = semantic_string.lower()
|
||||
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in semantic_lower for marker in required_markers):
|
||||
return True
|
||||
logger.debug(
|
||||
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
|
||||
f"{required_markers} but element has: {semantic_string}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Non-toggle intents pass through
|
||||
return True
|
||||
|
||||
@@ -138,6 +138,7 @@ class QNavGraph:
|
||||
"like": "tap like button",
|
||||
"comment": "tap comment button",
|
||||
"share": "tap share button",
|
||||
"follow": "tap follow button",
|
||||
}
|
||||
for keyword, required_action in action_checks.items():
|
||||
if keyword in goal.lower() and required_action not in available:
|
||||
|
||||
350
tests/e2e/test_follow_verification_integrity.py
Normal file
350
tests/e2e/test_follow_verification_integrity.py
Normal file
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
Follow Verification Integrity Tests — RED Phase (TDD)
|
||||
|
||||
These tests prove the LIES in our current test suite.
|
||||
Each one targets a specific gap that allowed the production bug:
|
||||
"🤝 [Follow] Followed @missiongreenenergy ✓" — when it actually clicked a photo grid item.
|
||||
|
||||
Root cause chain:
|
||||
1. VLM hallucinated a follow button (picked a photo)
|
||||
2. verify_success() asked the VLM again, VLM said "yes"
|
||||
3. No structural cross-check caught the mismatch
|
||||
4. FollowPlugin logged success based on nav_graph.do() return
|
||||
5. Qdrant memory was poisoned with a false positive
|
||||
|
||||
Each test MUST fail (RED) before any production code is fixed.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 1: verify_success MUST reject wrong-element clicks for follow
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestVerifySuccessRejectsWrongFollowElement:
|
||||
"""
|
||||
Production scenario: The bot clicked '3 photos by Mission Green Energy'
|
||||
instead of a Follow button. verify_success() should have caught this.
|
||||
|
||||
The VLM said "YES" because the screen changed (opening a photo).
|
||||
But the clicked element has NOTHING to do with 'follow'.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.memory = ActionMemory(ui_memory=MagicMock())
|
||||
|
||||
def test_follow_toggle_rejects_when_clicked_element_is_photo(self):
|
||||
"""
|
||||
If the tracked click was on a photo grid item (desc='3 photos by ...'),
|
||||
verify_success for 'follow' MUST return False — regardless of VLM opinion.
|
||||
|
||||
This is the ROOT CAUSE test. Today this passes because verify_success
|
||||
blindly trusts the VLM for toggle actions when confidence < 0.95.
|
||||
"""
|
||||
# Simulate what ActionMemory tracked before the click
|
||||
photo_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="3 photos by Mission Green Energy at row 1, column 3",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
self.memory.track_click("tap 'Follow' button", photo_node)
|
||||
|
||||
# The XML changed (photo opened), but the intent was 'follow'
|
||||
pre_xml = "<hierarchy><node resource-id='profile_tab'/></hierarchy>"
|
||||
post_xml = "<hierarchy><node resource-id='media_viewer'/></hierarchy>"
|
||||
|
||||
# The clicked element has NO relation to "follow" — desc is about photos
|
||||
# verify_success MUST detect this semantic mismatch structurally,
|
||||
# WITHOUT relying on VLM (which already lied once)
|
||||
result = self.memory.verify_success(
|
||||
"tap 'Follow' button",
|
||||
pre_xml,
|
||||
post_xml,
|
||||
device=None, # No device = no VLM fallback, pure structural
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# With device=None, it falls through to structural delta check.
|
||||
# Currently: diff > 0 for toggle → returns True (WRONG!)
|
||||
# The structural delta only checks length diff, not semantic match.
|
||||
#
|
||||
# This test PROVES the gap: a photo opening causes a structural delta,
|
||||
# which verify_success interprets as "follow succeeded".
|
||||
assert result is not True, (
|
||||
"CRITICAL: verify_success returned True for a follow intent "
|
||||
"when the clicked element was a PHOTO GRID ITEM! "
|
||||
"The structural delta falsely validated a screen change as 'follow success'."
|
||||
)
|
||||
|
||||
def test_follow_toggle_rejects_massive_structural_shift(self):
|
||||
"""
|
||||
When we click 'Follow', the XML should change minimally (button text changes).
|
||||
If the XML changes massively (>1000 chars), it means we navigated away.
|
||||
The current code DOES have this check, but only for diff > 1000.
|
||||
A photo view change can be 500-900 chars — slipping under the radar.
|
||||
"""
|
||||
pre_xml = "x" * 10000 # Simulated profile page
|
||||
post_xml = "y" * 10500 # Simulated photo view (500 chars diff)
|
||||
|
||||
photo_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="Photo by someone",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
self.memory.track_click("tap 'Follow' button", photo_node)
|
||||
|
||||
result = self.memory.verify_success(
|
||||
"tap 'Follow' button",
|
||||
pre_xml,
|
||||
post_xml,
|
||||
device=None,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# 500 chars diff is > 0 but < 1000, so current code returns True
|
||||
# But the CLICKED element was a photo, not a follow button!
|
||||
assert result is not True, (
|
||||
"verify_success accepted a 500-char structural delta for 'follow' "
|
||||
"without checking if the clicked element semantically matches the intent."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 2: QNavGraph.do() MUST block follow when no Follow button exists
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestQNavGraphDoBlocksFollowWithoutButton:
|
||||
"""
|
||||
QNavGraph.do() has screen-sanity checks for 'like', 'comment', 'share'
|
||||
but NOT for 'follow'. This means it blindly attempts to follow even
|
||||
when the current screen has no Follow button.
|
||||
"""
|
||||
|
||||
def test_do_rejects_follow_when_not_in_available_actions(self):
|
||||
"""
|
||||
If the current screen's available_actions does not contain 'tap follow button',
|
||||
QNavGraph.do("tap 'Follow' button") MUST return False immediately.
|
||||
|
||||
Currently: 'follow' is NOT in the action_checks map at q_nav_graph.py:L137-141,
|
||||
so it NEVER gets checked. The bot blindly passes through to GOAP.
|
||||
"""
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
device = MagicMock()
|
||||
device.dump_hierarchy.return_value = "<hierarchy/>"
|
||||
device.app_id = "com.instagram.android"
|
||||
|
||||
# Mock GOAP perceive to return a screen without 'follow' in available_actions
|
||||
mock_screen = {
|
||||
"screen_type": MagicMock(value="OTHER_PROFILE"),
|
||||
"available_actions": ["tap like button", "tap comment button", "scroll down"],
|
||||
}
|
||||
|
||||
with patch.object(QNavGraph, "__init__", lambda self, dev: None):
|
||||
nav = QNavGraph.__new__(QNavGraph)
|
||||
nav.device = device
|
||||
|
||||
mock_goap = MagicMock()
|
||||
mock_goap.perceive.return_value = mock_screen
|
||||
nav.goap = mock_goap
|
||||
|
||||
result = nav.do("tap 'Follow' button")
|
||||
|
||||
assert result is False, (
|
||||
"QNavGraph.do() allowed 'follow' to proceed without checking "
|
||||
"if 'tap follow button' is in available_actions! "
|
||||
"The action_checks map at q_nav_graph.py:L137 is missing 'follow'."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 3: ActionMemory.confirm_click() MUST NOT poison Qdrant with mismatched intents
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestActionMemoryNeverConfirmsMismatch:
|
||||
"""
|
||||
After a false VLM verification, confirm_click() stores the wrong
|
||||
click mapping in Qdrant. Next time the bot sees this intent,
|
||||
it will recall the photo grid item instead of looking for Follow.
|
||||
"""
|
||||
|
||||
def test_confirm_click_rejects_semantic_mismatch(self):
|
||||
"""
|
||||
If track_click recorded intent='tap Follow button' but the node
|
||||
is desc='3 photos by Mission Green Energy', confirm_click()
|
||||
MUST refuse to store this in Qdrant.
|
||||
|
||||
Currently: confirm_click() blindly stores whatever was tracked,
|
||||
poisoning the memory DB.
|
||||
"""
|
||||
mock_ui_memory = MagicMock()
|
||||
mock_ui_memory.retrieve_memory.return_value = None
|
||||
memory = ActionMemory(ui_memory=mock_ui_memory)
|
||||
|
||||
# Track a click on the WRONG element
|
||||
wrong_node = SpatialNode(
|
||||
resource_id="com.instagram.android:id/image_button",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="3 photos by Mission Green Energy at row 1, column 3",
|
||||
bounds=(0, 400, 360, 760),
|
||||
clickable=True,
|
||||
)
|
||||
memory.track_click("tap 'Follow' button", wrong_node)
|
||||
|
||||
# Production flow calls confirm_click after VLM says "yes"
|
||||
memory.confirm_click("tap 'Follow' button")
|
||||
|
||||
# Qdrant store_memory should NOT have been called because
|
||||
# the element has nothing to do with 'follow'
|
||||
assert not mock_ui_memory.store_memory.called, (
|
||||
"CRITICAL: ActionMemory.confirm_click() stored a PHOTO GRID ITEM "
|
||||
"as the successful click target for 'tap Follow button'! "
|
||||
"This poisons Qdrant and causes the same wrong click on every future run."
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 4: GOAP interaction path MUST cross-check clicked element vs intent
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestGOAPInteractionCrossCheck:
|
||||
"""
|
||||
GOAP._execute_action() trusts VLM twice:
|
||||
1. VLM selects the element to click
|
||||
2. VLM verifies if the click was successful
|
||||
|
||||
If VLM #1 hallucinated, VLM #2 will also lie (confirmation bias).
|
||||
There MUST be a structural cross-check between the selected element
|
||||
and the intent BEFORE trusting the VLM verification.
|
||||
"""
|
||||
|
||||
def test_execute_action_rejects_when_clicked_node_doesnt_match_intent(self):
|
||||
"""
|
||||
If find_best_node returns a node with desc='3 photos by ...'
|
||||
for intent='tap Follow button', _execute_action MUST reject it
|
||||
BEFORE even clicking.
|
||||
|
||||
Currently: _execute_action clicks first, then asks VLM to verify.
|
||||
The VLM verification is the fox guarding the henhouse.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
device = MagicMock()
|
||||
device.dump_hierarchy.return_value = "<hierarchy/>"
|
||||
device.app_id = "com.instagram.android"
|
||||
|
||||
executor = GoalExecutor(device, bot_username="testbot")
|
||||
|
||||
# Mock TelepathicEngine to return a photo node for a follow intent
|
||||
mock_node = {
|
||||
"x": 180,
|
||||
"y": 580,
|
||||
"text": "",
|
||||
"description": "3 photos by Mission Green Energy at row 1, column 3",
|
||||
"id": "com.instagram.android:id/image_button",
|
||||
"class": "android.widget.ImageView",
|
||||
"score": 0.7,
|
||||
}
|
||||
|
||||
with patch("GramAddict.core.telepathic_engine.TelepathicEngine") as MockTE:
|
||||
mock_engine = MagicMock()
|
||||
MockTE.get_instance.return_value = mock_engine
|
||||
mock_engine.find_best_node.return_value = mock_node
|
||||
|
||||
# Mock perceive to return a dummy screen state
|
||||
executor.screen_id = MagicMock()
|
||||
executor.screen_id.identify.return_value = {
|
||||
"screen_type": MagicMock(value="OTHER_PROFILE"),
|
||||
"available_actions": [],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
result = executor._execute_action("tap 'Follow' button")
|
||||
|
||||
# The method should have rejected this node BEFORE clicking
|
||||
assert result is False, (
|
||||
"GOAP._execute_action accepted a PHOTO GRID ITEM for 'tap Follow button'! "
|
||||
"There is no pre-click sanity check that the selected node "
|
||||
"semantically matches the intent."
|
||||
)
|
||||
# Verify that device.click was NOT called
|
||||
device.click.assert_not_called()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# TEST 5: FollowPlugin.execute() E2E — end-to-end truth test
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestFollowPluginEndToEnd:
|
||||
"""
|
||||
The most critical gap: FollowPlugin.execute() is never tested E2E.
|
||||
It calls nav_graph.do("tap 'Follow' button") and trusts the boolean.
|
||||
If do() lies (returns True when it clicked a photo), the entire
|
||||
session state is corrupted.
|
||||
"""
|
||||
|
||||
def test_follow_plugin_does_not_count_follow_when_wrong_element_clicked(self):
|
||||
"""
|
||||
If nav_graph.do() returns True but actually clicked a photo,
|
||||
the session_state.add_interaction(followed=True) poisons the stats.
|
||||
|
||||
This test proves that FollowPlugin has ZERO verification of its own.
|
||||
It blindly trusts nav_graph.do().
|
||||
"""
|
||||
from GramAddict.core.behaviors import BehaviorContext
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
plugin = FollowPlugin()
|
||||
|
||||
# Build a minimal BehaviorContext
|
||||
mock_session = MagicMock()
|
||||
mock_configs = MagicMock()
|
||||
mock_configs.args.follow_percentage = 100
|
||||
plugin.get_config = MagicMock(return_value={"percentage": "100"})
|
||||
|
||||
mock_nav = MagicMock()
|
||||
# nav_graph.do() returns True (the lie)
|
||||
mock_nav.do.return_value = True
|
||||
|
||||
ctx = BehaviorContext(
|
||||
device=MagicMock(),
|
||||
session_state=mock_session,
|
||||
configs=mock_configs,
|
||||
username="missiongreenenergy",
|
||||
cognitive_stack={"nav_graph": mock_nav},
|
||||
)
|
||||
|
||||
result = plugin.execute(ctx)
|
||||
|
||||
# The plugin MUST have some way to verify the follow actually happened.
|
||||
# Currently it doesn't — it just checks `if nav_graph.do(...)`.
|
||||
# This test documents the gap: if do() lies, so does the plugin.
|
||||
#
|
||||
# At minimum, the plugin should check that the post-click screen
|
||||
# shows "Following" or "Requested" instead of blindly trusting do().
|
||||
assert result.executed is True, "Expected plugin to report executed (it trusts do())"
|
||||
|
||||
# But HERE is the real assertion: the session state should NOT record
|
||||
# a follow if there's no structural proof the follow happened.
|
||||
# This proves the plugin has no independent verification.
|
||||
mock_session.add_interaction.assert_called_once()
|
||||
call_kwargs = mock_session.add_interaction.call_args
|
||||
assert call_kwargs[1].get("followed") is True or call_kwargs.kwargs.get("followed") is True, (
|
||||
"Plugin recorded followed=True — but it has NO independent verification! "
|
||||
"This test documents the architectural gap: FollowPlugin blindly trusts QNavGraph.do()."
|
||||
)
|
||||
Reference in New Issue
Block a user