fix(follow+vlm): prevent following-click catastrophe & stop VLM memory poisoning
FOLLOW PLUGIN SAFETY GUARD: - Added pre-click XML guard: checks button text before clicking - If button says 'Following'/'Requested'/'Message' → SKIP (prevents opening dangerous Unfollow/Add-to-Favorites bottom sheet) - Changed intent from 'tap Follow or Following button' to 'tap follow button' - This was the root cause of adding users to Close Friends/Favorites VLM VERIFICATION RESILIENCE: - VLM false no longer shortcircuits to return False - VLM verdict is now a SOFT SIGNAL that falls through to structural delta verification as ground-truth tiebreaker - Small local VLMs (7B llava) systematically return false for everything - This was THE root cause of memory poisoning: every action got penalized because VLM always said 'failed', regardless of actual screen state - 22 total poisoned Qdrant entries purged across two sessions ROOT CAUSE CHAIN: VLM always says false → penalties on every action → confidence < 0.2 → circuit breaker blocks ALL core actions → bot can't like/follow/comment → bot clicks random things → adds to favorites, opens bottom sheets TDD: 4 new tests, full suite 110/111 (1 infra flake)
This commit is contained in:
124
tests/tdd/test_follow_and_vlm.py
Normal file
124
tests/tdd/test_follow_and_vlm.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
TDD: Follow Plugin Safety Guard + VLM Verification Resilience
|
||||
|
||||
1. Follow plugin must REFUSE to click "Following" buttons (already followed).
|
||||
Only "Follow" buttons (not yet followed) are valid targets.
|
||||
2. VLM verification must NOT be the sole source of truth. When VLM says "false"
|
||||
but structural delta shows significant UI change, structural evidence wins.
|
||||
3. VLM false negatives must NOT poison the memory system.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ── Follow Plugin Tests ──
|
||||
|
||||
|
||||
class TestFollowPluginSafetyGuard:
|
||||
"""The follow plugin must distinguish 'Follow' from 'Following' buttons."""
|
||||
|
||||
def test_follow_plugin_intent_only_targets_follow_not_following(self):
|
||||
"""
|
||||
META-TEST: The follow plugin's intent must specifically target ONLY
|
||||
the "Follow" state (unfollowed users), not the "Following" state.
|
||||
Clicking "Following" opens an unfollow/favorites bottom sheet.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
source = inspect.getsource(FollowPlugin.execute)
|
||||
|
||||
# The intent must NOT be "tap 'Follow' or 'Following' button"
|
||||
# because that would match both states
|
||||
assert "'Follow' or 'Following'" not in source, (
|
||||
"Follow plugin must NOT use an intent that matches both 'Follow' AND 'Following'. "
|
||||
"Clicking 'Following' opens a dangerous bottom sheet (Unfollow/Add to Favorites/Close Friends). "
|
||||
"The intent must ONLY target the 'Follow' state."
|
||||
)
|
||||
|
||||
def test_follow_plugin_has_already_following_guard(self):
|
||||
"""
|
||||
RED: Follow plugin must check if the button already says 'Following'
|
||||
and SKIP if so. This prevents opening the dangerous bottom sheet.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
source = inspect.getsource(FollowPlugin.execute)
|
||||
source_lower = source.lower()
|
||||
|
||||
# Must have some guard that checks for "following" state
|
||||
has_following_guard = (
|
||||
"following" in source_lower and
|
||||
("skip" in source_lower or "already" in source_lower or "abort" in source_lower or "return" in source_lower)
|
||||
)
|
||||
assert has_following_guard, (
|
||||
"Follow plugin must guard against clicking 'Following' buttons. "
|
||||
"It must check the button text and SKIP if the user is already followed."
|
||||
)
|
||||
|
||||
|
||||
# ── VLM Verification Resilience Tests ──
|
||||
|
||||
|
||||
class TestVLMVerificationResilience:
|
||||
"""
|
||||
VLM verification must NOT be the sole authority on action success.
|
||||
Structural evidence must be able to OVERRIDE a VLM false negative.
|
||||
"""
|
||||
|
||||
def test_vlm_false_does_not_shortcircuit_when_structural_delta_exists(self):
|
||||
"""
|
||||
META-TEST: When VLM says false but there IS significant structural
|
||||
delta, the verification must NOT blindly return False.
|
||||
Structural evidence must be considered.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
source = inspect.getsource(ActionMemory.verify_success)
|
||||
|
||||
# Find the VLM false block — it must NOT just "return False"
|
||||
# It should fallthrough to structural verification
|
||||
lines = source.split("\n")
|
||||
vlm_false_section = False
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if "decision is False" in stripped or "decision == False" in stripped:
|
||||
vlm_false_section = True
|
||||
continue
|
||||
if vlm_false_section:
|
||||
# The very next meaningful line after detecting VLM false
|
||||
# must NOT be a hard "return False"
|
||||
if stripped and not stripped.startswith("#") and not stripped.startswith('"""'):
|
||||
assert "return False" not in stripped, (
|
||||
f"Line {i}: VLM returning False must NOT shortcircuit to 'return False'. "
|
||||
"It must fallthrough to structural delta verification. "
|
||||
"A small local VLM (7B) is unreliable and systematically poisons "
|
||||
"the memory by saying everything failed."
|
||||
)
|
||||
break # Only check the first meaningful line after
|
||||
|
||||
def test_verify_success_has_vlm_structural_reconciliation(self):
|
||||
"""
|
||||
META-TEST: verify_success must have logic that reconciles VLM verdicts
|
||||
with structural delta evidence. If structural delta shows significant
|
||||
change (>50 chars), VLM false should be overridden.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
source = inspect.getsource(ActionMemory.verify_success)
|
||||
|
||||
# Must have some concept of VLM being unreliable or structural override
|
||||
has_reconciliation = (
|
||||
"structural" in source.lower() and
|
||||
("override" in source.lower() or "fallthrough" in source.lower() or
|
||||
"vlm_verdict" in source.lower() or "vlm_decision" in source.lower() or
|
||||
"trust" in source.lower())
|
||||
)
|
||||
assert has_reconciliation, (
|
||||
"verify_success must reconcile VLM verdicts with structural evidence. "
|
||||
"A VLM false-negative when structural delta shows significant change "
|
||||
"should not be treated as a hard failure."
|
||||
)
|
||||
Reference in New Issue
Block a user