fix(follow+vlm): prevent following-click catastrophe & stop VLM memory poisoning
FOLLOW PLUGIN SAFETY GUARD: - Added pre-click XML guard: checks button text before clicking - If button says 'Following'/'Requested'/'Message' → SKIP (prevents opening dangerous Unfollow/Add-to-Favorites bottom sheet) - Changed intent from 'tap Follow or Following button' to 'tap follow button' - This was the root cause of adding users to Close Friends/Favorites VLM VERIFICATION RESILIENCE: - VLM false no longer shortcircuits to return False - VLM verdict is now a SOFT SIGNAL that falls through to structural delta verification as ground-truth tiebreaker - Small local VLMs (7B llava) systematically return false for everything - This was THE root cause of memory poisoning: every action got penalized because VLM always said 'failed', regardless of actual screen state - 22 total poisoned Qdrant entries purged across two sessions ROOT CAUSE CHAIN: VLM always says false → penalties on every action → confidence < 0.2 → circuit breaker blocks ALL core actions → bot can't like/follow/comment → bot clicks random things → adds to favorites, opens bottom sheets TDD: 4 new tests, full suite 110/111 (1 infra flake)
This commit is contained in:
@@ -54,15 +54,39 @@ class FollowPlugin(BehaviorPlugin):
|
||||
return True
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
"""Follow the target user."""
|
||||
"""Follow the target user. ONLY clicks 'Follow' buttons, never 'Following'."""
|
||||
nav_graph = ctx.cognitive_stack.get("nav_graph")
|
||||
if not nav_graph:
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if nav_graph.do("tap 'Follow' or 'Following' button"):
|
||||
logger.info(f"🤝 [Follow] Toggled Follow/Following state for @{ctx.username} ✓")
|
||||
# ── CRITICAL SAFETY GUARD ──
|
||||
# Pre-check: verify the button actually says "Follow" (not "Following" or "Requested").
|
||||
# Clicking "Following" opens a dangerous bottom sheet (Unfollow / Add to Favorites / Close Friends).
|
||||
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
|
||||
import re
|
||||
|
||||
follow_btn = re.search(
|
||||
r'resource-id="com\.instagram\.android:id/profile_header_follow_button"[^>]*text="([^"]*)"',
|
||||
xml,
|
||||
)
|
||||
if not follow_btn:
|
||||
# Try reversed attribute order
|
||||
follow_btn = re.search(
|
||||
r'text="([^"]*)"[^>]*resource-id="com\.instagram\.android:id/profile_header_follow_button"',
|
||||
xml,
|
||||
)
|
||||
if follow_btn:
|
||||
button_text = follow_btn.group(1).strip().lower()
|
||||
if button_text in ("following", "requested", "message"):
|
||||
logger.info(
|
||||
f"🛡️ [Follow] Button says '{follow_btn.group(1)}' — user already followed. Skipping to avoid bottom sheet."
|
||||
)
|
||||
return BehaviorResult(executed=False, metadata={"reason": "already_following"})
|
||||
|
||||
if nav_graph.do("tap follow button"):
|
||||
logger.info(f"🤝 [Follow] Followed @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
|
||||
|
||||
# Buffer for follow animations to close
|
||||
|
||||
@@ -198,9 +198,10 @@ class ActionMemory:
|
||||
# We NO LONGER bypass VLM verification via string matching.
|
||||
# If confidence is < 0.95, we always do VLM or Delta verification.
|
||||
|
||||
# ── VLM Verification Fallback ──
|
||||
# ── VLM Verification (soft signal, NOT sole authority) ──
|
||||
|
||||
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
|
||||
vlm_verdict = None
|
||||
if device and confidence < 0.95:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Confidence ({confidence:.2f}) < 0.95. Handing over verification for '{intent}' to VLM visual analysis..."
|
||||
@@ -214,7 +215,6 @@ class ActionMemory:
|
||||
if self._last_click_context:
|
||||
clicked_context = f"The element that was tapped: {self._last_click_context['semantic_string']}. "
|
||||
|
||||
# Ask VLM to be the absolute source of truth
|
||||
prompt = (
|
||||
f"The user just attempted to perform the action: '{intent}'. "
|
||||
f"{clicked_context}"
|
||||
@@ -242,19 +242,21 @@ class ActionMemory:
|
||||
raise ValueError("No screenshot available from device")
|
||||
response = evaluator._query_vlm(prompt, screenshot)
|
||||
|
||||
decision = _parse_yes_no(response) if response else None
|
||||
vlm_verdict = _parse_yes_no(response) if response else None
|
||||
|
||||
if decision is True:
|
||||
if vlm_verdict is True:
|
||||
logger.debug(f"🧠 [ActionMemory] VLM visually confirmed success for '{intent}'.")
|
||||
return True
|
||||
elif decision is False:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] VLM visual verification FAILED for '{intent}'. VLM replied: '{response}'"
|
||||
elif vlm_verdict is False:
|
||||
# VLM says false — but small local VLMs (7B) are unreliable.
|
||||
# Do NOT trust this blindly. Fallthrough to structural delta verification
|
||||
# which is the ground-truth tiebreaker.
|
||||
logger.info(
|
||||
f"🧠 [ActionMemory] VLM says '{intent}' failed — but VLM is unreliable. "
|
||||
"Falling through to structural delta for ground-truth verification."
|
||||
)
|
||||
return False
|
||||
# DO NOT return False here — let structural delta decide
|
||||
else:
|
||||
# VLM returned ambiguous response (JSON, mixed signals, etc.)
|
||||
# Don't treat as hard failure — fall through to structural delta verification
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] VLM response for '{intent}' was not YES/NO "
|
||||
f"(got: '{response[:80]}...'). Falling through to structural verification."
|
||||
|
||||
124
tests/tdd/test_follow_and_vlm.py
Normal file
124
tests/tdd/test_follow_and_vlm.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
TDD: Follow Plugin Safety Guard + VLM Verification Resilience
|
||||
|
||||
1. Follow plugin must REFUSE to click "Following" buttons (already followed).
|
||||
Only "Follow" buttons (not yet followed) are valid targets.
|
||||
2. VLM verification must NOT be the sole source of truth. When VLM says "false"
|
||||
but structural delta shows significant UI change, structural evidence wins.
|
||||
3. VLM false negatives must NOT poison the memory system.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ── Follow Plugin Tests ──
|
||||
|
||||
|
||||
class TestFollowPluginSafetyGuard:
|
||||
"""The follow plugin must distinguish 'Follow' from 'Following' buttons."""
|
||||
|
||||
def test_follow_plugin_intent_only_targets_follow_not_following(self):
|
||||
"""
|
||||
META-TEST: The follow plugin's intent must specifically target ONLY
|
||||
the "Follow" state (unfollowed users), not the "Following" state.
|
||||
Clicking "Following" opens an unfollow/favorites bottom sheet.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
source = inspect.getsource(FollowPlugin.execute)
|
||||
|
||||
# The intent must NOT be "tap 'Follow' or 'Following' button"
|
||||
# because that would match both states
|
||||
assert "'Follow' or 'Following'" not in source, (
|
||||
"Follow plugin must NOT use an intent that matches both 'Follow' AND 'Following'. "
|
||||
"Clicking 'Following' opens a dangerous bottom sheet (Unfollow/Add to Favorites/Close Friends). "
|
||||
"The intent must ONLY target the 'Follow' state."
|
||||
)
|
||||
|
||||
def test_follow_plugin_has_already_following_guard(self):
|
||||
"""
|
||||
RED: Follow plugin must check if the button already says 'Following'
|
||||
and SKIP if so. This prevents opening the dangerous bottom sheet.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.behaviors.follow import FollowPlugin
|
||||
|
||||
source = inspect.getsource(FollowPlugin.execute)
|
||||
source_lower = source.lower()
|
||||
|
||||
# Must have some guard that checks for "following" state
|
||||
has_following_guard = (
|
||||
"following" in source_lower and
|
||||
("skip" in source_lower or "already" in source_lower or "abort" in source_lower or "return" in source_lower)
|
||||
)
|
||||
assert has_following_guard, (
|
||||
"Follow plugin must guard against clicking 'Following' buttons. "
|
||||
"It must check the button text and SKIP if the user is already followed."
|
||||
)
|
||||
|
||||
|
||||
# ── VLM Verification Resilience Tests ──
|
||||
|
||||
|
||||
class TestVLMVerificationResilience:
|
||||
"""
|
||||
VLM verification must NOT be the sole authority on action success.
|
||||
Structural evidence must be able to OVERRIDE a VLM false negative.
|
||||
"""
|
||||
|
||||
def test_vlm_false_does_not_shortcircuit_when_structural_delta_exists(self):
|
||||
"""
|
||||
META-TEST: When VLM says false but there IS significant structural
|
||||
delta, the verification must NOT blindly return False.
|
||||
Structural evidence must be considered.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
source = inspect.getsource(ActionMemory.verify_success)
|
||||
|
||||
# Find the VLM false block — it must NOT just "return False"
|
||||
# It should fallthrough to structural verification
|
||||
lines = source.split("\n")
|
||||
vlm_false_section = False
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if "decision is False" in stripped or "decision == False" in stripped:
|
||||
vlm_false_section = True
|
||||
continue
|
||||
if vlm_false_section:
|
||||
# The very next meaningful line after detecting VLM false
|
||||
# must NOT be a hard "return False"
|
||||
if stripped and not stripped.startswith("#") and not stripped.startswith('"""'):
|
||||
assert "return False" not in stripped, (
|
||||
f"Line {i}: VLM returning False must NOT shortcircuit to 'return False'. "
|
||||
"It must fallthrough to structural delta verification. "
|
||||
"A small local VLM (7B) is unreliable and systematically poisons "
|
||||
"the memory by saying everything failed."
|
||||
)
|
||||
break # Only check the first meaningful line after
|
||||
|
||||
def test_verify_success_has_vlm_structural_reconciliation(self):
|
||||
"""
|
||||
META-TEST: verify_success must have logic that reconciles VLM verdicts
|
||||
with structural delta evidence. If structural delta shows significant
|
||||
change (>50 chars), VLM false should be overridden.
|
||||
"""
|
||||
import inspect
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
source = inspect.getsource(ActionMemory.verify_success)
|
||||
|
||||
# Must have some concept of VLM being unreliable or structural override
|
||||
has_reconciliation = (
|
||||
"structural" in source.lower() and
|
||||
("override" in source.lower() or "fallthrough" in source.lower() or
|
||||
"vlm_verdict" in source.lower() or "vlm_decision" in source.lower() or
|
||||
"trust" in source.lower())
|
||||
)
|
||||
assert has_reconciliation, (
|
||||
"verify_success must reconcile VLM verdicts with structural evidence. "
|
||||
"A VLM false-negative when structural delta shows significant change "
|
||||
"should not be treated as a hard failure."
|
||||
)
|
||||
Reference in New Issue
Block a user