fix(follow+vlm): prevent following-click catastrophe & stop VLM memory poisoning
FOLLOW PLUGIN SAFETY GUARD: - Added pre-click XML guard: checks button text before clicking - If button says 'Following'/'Requested'/'Message' → SKIP (prevents opening dangerous Unfollow/Add-to-Favorites bottom sheet) - Changed intent from 'tap Follow or Following button' to 'tap follow button' - This was the root cause of adding users to Close Friends/Favorites VLM VERIFICATION RESILIENCE: - VLM false no longer shortcircuits to return False - VLM verdict is now a SOFT SIGNAL that falls through to structural delta verification as ground-truth tiebreaker - Small local VLMs (7B llava) systematically return false for everything - This was THE root cause of memory poisoning: every action got penalized because VLM always said 'failed', regardless of actual screen state - 22 total poisoned Qdrant entries purged across two sessions ROOT CAUSE CHAIN: VLM always says false → penalties on every action → confidence < 0.2 → circuit breaker blocks ALL core actions → bot can't like/follow/comment → bot clicks random things → adds to favorites, opens bottom sheets TDD: 4 new tests, full suite 110/111 (1 infra flake)
This commit is contained in:
@@ -54,15 +54,39 @@ class FollowPlugin(BehaviorPlugin):
|
||||
return True
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
"""Follow the target user."""
|
||||
"""Follow the target user. ONLY clicks 'Follow' buttons, never 'Following'."""
|
||||
nav_graph = ctx.cognitive_stack.get("nav_graph")
|
||||
if not nav_graph:
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if nav_graph.do("tap 'Follow' or 'Following' button"):
|
||||
logger.info(f"🤝 [Follow] Toggled Follow/Following state for @{ctx.username} ✓")
|
||||
# ── CRITICAL SAFETY GUARD ──
|
||||
# Pre-check: verify the button actually says "Follow" (not "Following" or "Requested").
|
||||
# Clicking "Following" opens a dangerous bottom sheet (Unfollow / Add to Favorites / Close Friends).
|
||||
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
|
||||
import re
|
||||
|
||||
follow_btn = re.search(
|
||||
r'resource-id="com\.instagram\.android:id/profile_header_follow_button"[^>]*text="([^"]*)"',
|
||||
xml,
|
||||
)
|
||||
if not follow_btn:
|
||||
# Try reversed attribute order
|
||||
follow_btn = re.search(
|
||||
r'text="([^"]*)"[^>]*resource-id="com\.instagram\.android:id/profile_header_follow_button"',
|
||||
xml,
|
||||
)
|
||||
if follow_btn:
|
||||
button_text = follow_btn.group(1).strip().lower()
|
||||
if button_text in ("following", "requested", "message"):
|
||||
logger.info(
|
||||
f"🛡️ [Follow] Button says '{follow_btn.group(1)}' — user already followed. Skipping to avoid bottom sheet."
|
||||
)
|
||||
return BehaviorResult(executed=False, metadata={"reason": "already_following"})
|
||||
|
||||
if nav_graph.do("tap follow button"):
|
||||
logger.info(f"🤝 [Follow] Followed @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
|
||||
|
||||
# Buffer for follow animations to close
|
||||
|
||||
@@ -198,9 +198,10 @@ class ActionMemory:
|
||||
# We NO LONGER bypass VLM verification via string matching.
|
||||
# If confidence is < 0.95, we always do VLM or Delta verification.
|
||||
|
||||
# ── VLM Verification Fallback ──
|
||||
# ── VLM Verification (soft signal, NOT sole authority) ──
|
||||
|
||||
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
|
||||
vlm_verdict = None
|
||||
if device and confidence < 0.95:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Confidence ({confidence:.2f}) < 0.95. Handing over verification for '{intent}' to VLM visual analysis..."
|
||||
@@ -214,7 +215,6 @@ class ActionMemory:
|
||||
if self._last_click_context:
|
||||
clicked_context = f"The element that was tapped: {self._last_click_context['semantic_string']}. "
|
||||
|
||||
# Ask VLM to be the absolute source of truth
|
||||
prompt = (
|
||||
f"The user just attempted to perform the action: '{intent}'. "
|
||||
f"{clicked_context}"
|
||||
@@ -242,19 +242,21 @@ class ActionMemory:
|
||||
raise ValueError("No screenshot available from device")
|
||||
response = evaluator._query_vlm(prompt, screenshot)
|
||||
|
||||
decision = _parse_yes_no(response) if response else None
|
||||
vlm_verdict = _parse_yes_no(response) if response else None
|
||||
|
||||
if decision is True:
|
||||
if vlm_verdict is True:
|
||||
logger.debug(f"🧠 [ActionMemory] VLM visually confirmed success for '{intent}'.")
|
||||
return True
|
||||
elif decision is False:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] VLM visual verification FAILED for '{intent}'. VLM replied: '{response}'"
|
||||
elif vlm_verdict is False:
|
||||
# VLM says false — but small local VLMs (7B) are unreliable.
|
||||
# Do NOT trust this blindly. Fallthrough to structural delta verification
|
||||
# which is the ground-truth tiebreaker.
|
||||
logger.info(
|
||||
f"🧠 [ActionMemory] VLM says '{intent}' failed — but VLM is unreliable. "
|
||||
"Falling through to structural delta for ground-truth verification."
|
||||
)
|
||||
return False
|
||||
# DO NOT return False here — let structural delta decide
|
||||
else:
|
||||
# VLM returned ambiguous response (JSON, mixed signals, etc.)
|
||||
# Don't treat as hard failure — fall through to structural delta verification
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] VLM response for '{intent}' was not YES/NO "
|
||||
f"(got: '{response[:80]}...'). Falling through to structural verification."
|
||||
|
||||
Reference in New Issue
Block a user