fix(follow+vlm): prevent following-click catastrophe & stop VLM memory poisoning

FOLLOW PLUGIN SAFETY GUARD:
- Added pre-click XML guard: checks button text before clicking
- If button says 'Following'/'Requested'/'Message' → SKIP
  (prevents opening dangerous Unfollow/Add-to-Favorites bottom sheet)
- Changed intent from 'tap Follow or Following button' to 'tap follow button'
- This was the root cause of adding users to Close Friends/Favorites

VLM VERIFICATION RESILIENCE:
- VLM false no longer shortcircuits to return False
- VLM verdict is now a SOFT SIGNAL that falls through to structural
  delta verification as ground-truth tiebreaker
- Small local VLMs (7B llava) systematically return false for everything
- This was THE root cause of memory poisoning: every action got penalized
  because VLM always said 'failed', regardless of actual screen state
- 22 total poisoned Qdrant entries purged across two sessions

ROOT CAUSE CHAIN:
VLM always says false → penalties on every action → confidence < 0.2 →
circuit breaker blocks ALL core actions → bot can't like/follow/comment
→ bot clicks random things → adds to favorites, opens bottom sheets

TDD: 4 new tests, full suite 110/111 (1 infra flake)
This commit is contained in:
2026-05-05 16:05:31 +02:00
parent 749e82ea14
commit 8a2c93fb64
3 changed files with 163 additions and 13 deletions

View File

@@ -54,15 +54,39 @@ class FollowPlugin(BehaviorPlugin):
return True
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
"""Follow the target user."""
"""Follow the target user. ONLY clicks 'Follow' buttons, never 'Following'."""
nav_graph = ctx.cognitive_stack.get("nav_graph")
if not nav_graph:
from GramAddict.core.q_nav_graph import QNavGraph
nav_graph = QNavGraph(ctx.device)
if nav_graph.do("tap 'Follow' or 'Following' button"):
logger.info(f"🤝 [Follow] Toggled Follow/Following state for @{ctx.username}")
# ── CRITICAL SAFETY GUARD ──
# Pre-check: verify the button actually says "Follow" (not "Following" or "Requested").
# Clicking "Following" opens a dangerous bottom sheet (Unfollow / Add to Favorites / Close Friends).
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
import re
follow_btn = re.search(
r'resource-id="com\.instagram\.android:id/profile_header_follow_button"[^>]*text="([^"]*)"',
xml,
)
if not follow_btn:
# Try reversed attribute order
follow_btn = re.search(
r'text="([^"]*)"[^>]*resource-id="com\.instagram\.android:id/profile_header_follow_button"',
xml,
)
if follow_btn:
button_text = follow_btn.group(1).strip().lower()
if button_text in ("following", "requested", "message"):
logger.info(
f"🛡️ [Follow] Button says '{follow_btn.group(1)}' — user already followed. Skipping to avoid bottom sheet."
)
return BehaviorResult(executed=False, metadata={"reason": "already_following"})
if nav_graph.do("tap follow button"):
logger.info(f"🤝 [Follow] Followed @{ctx.username}")
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
# Buffer for follow animations to close

View File

@@ -198,9 +198,10 @@ class ActionMemory:
# We NO LONGER bypass VLM verification via string matching.
# If confidence is < 0.95, we always do VLM or Delta verification.
# ── VLM Verification Fallback ──
# ── VLM Verification (soft signal, NOT sole authority) ──
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
vlm_verdict = None
if device and confidence < 0.95:
logger.info(
f"👁️ [ActionMemory] Confidence ({confidence:.2f}) < 0.95. Handing over verification for '{intent}' to VLM visual analysis..."
@@ -214,7 +215,6 @@ class ActionMemory:
if self._last_click_context:
clicked_context = f"The element that was tapped: {self._last_click_context['semantic_string']}. "
# Ask VLM to be the absolute source of truth
prompt = (
f"The user just attempted to perform the action: '{intent}'. "
f"{clicked_context}"
@@ -242,19 +242,21 @@ class ActionMemory:
raise ValueError("No screenshot available from device")
response = evaluator._query_vlm(prompt, screenshot)
decision = _parse_yes_no(response) if response else None
vlm_verdict = _parse_yes_no(response) if response else None
if decision is True:
if vlm_verdict is True:
logger.debug(f"🧠 [ActionMemory] VLM visually confirmed success for '{intent}'.")
return True
elif decision is False:
logger.warning(
f"⚠️ [ActionMemory] VLM visual verification FAILED for '{intent}'. VLM replied: '{response}'"
elif vlm_verdict is False:
# VLM says false — but small local VLMs (7B) are unreliable.
# Do NOT trust this blindly. Fallthrough to structural delta verification
# which is the ground-truth tiebreaker.
logger.info(
f"🧠 [ActionMemory] VLM says '{intent}' failed — but VLM is unreliable. "
"Falling through to structural delta for ground-truth verification."
)
return False
# DO NOT return False here — let structural delta decide
else:
# VLM returned ambiguous response (JSON, mixed signals, etc.)
# Don't treat as hard failure — fall through to structural delta verification
logger.debug(
f"🧠 [ActionMemory] VLM response for '{intent}' was not YES/NO "
f"(got: '{response[:80]}...'). Falling through to structural verification."