mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(perception): structural Resource-ID bypass gate + dead code purge in ActionMemory

Browse Source 
P0-1: Toggle intents (follow/like/save) resolved via Resource-ID fast-path
now skip VLM verification entirely. Eliminates the #1 session failure:
VLM hallucinating Follow→Like, causing zero follows per session.

P0-2: Purge 52 lines of unreachable dead code after unconditional return
at line 290. Non-toggle structural delta verification now properly reachable.
Remove 4 DEBUG-prefixed logger.info pollution statements.

TDD: 7 new tests covering bypass gate, negative guard, non-toggle delta,
and debug log pollution detection.
This commit is contained in:
Marc Mintel
2026-05-04 14:35:19 +02:00
parent f384fbb749
commit d81a5d4e81
2 changed files with 254 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
GramAddict/core/perception/action_memory.py
View File

@@ -196,6 +196,23 @@ class ActionMemory:
state_toggles = ["like", "save", "follow", "heart"]
is_toggle = any(t in intent_lower for t in state_toggles)
# ── P0-1: Structural Resource-ID Bypass Gate ──
# If the clicked node was resolved via a structural Resource-ID that
# directly matches the toggle intent, VLM verification is SKIPPED.
# This eliminates the #1 session failure: VLM hallucinating Follow→Like.
if is_toggle and self._last_click_context:
clicked_rid = (self._last_click_context.get("node_dict", {}).get("resource_id", "") or "").lower()
if clicked_rid:
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
if intent_keyword in intent_lower:
if any(marker in clicked_rid for marker in required_markers):
logger.info(
f"⚡ [ActionMemory] Structural Resource-ID bypass: '{intent}' matched "
f"'{clicked_rid}'. Skipping VLM verification — O(1) trust."
)
return True
break # Only check the first matching intent keyword
# ── VLM Verification Fallback ──
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
@@ -270,10 +287,8 @@ class ActionMemory:
)
return False
# Fallback to structural delta
logger.info(f"DEBUG: len(pre_click_xml)={len(pre_click_xml)} len(post_click_xml)={len(post_click_xml)}")
# ── Structural Delta Verification ──
diff = abs(len(pre_click_xml) - len(post_click_xml))
logger.info(f"DEBUG: diff={diff}")
if is_toggle:
if diff > 1000:
@@ -288,58 +303,18 @@ class ActionMemory:
f"⚠️ [ActionMemory] Zero structural shift (diff={diff}) for state-toggle '{intent}'. Verification FAIL."
)
return False
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
# also produces a large diff but achieves the wrong goal.
if diff > 50:
# Is it a standard structural transition?
from GramAddict.core.screen_topology import ScreenTopology
# We don't have screen type here, so we just check if it's in the HD Map keys
logger.info(f"DEBUG: intent is '{intent}'")
logger.info(
f"DEBUG: TRANSITIONS keys are: {[list(t.keys()) for t in ScreenTopology.TRANSITIONS.values()]}"
)
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
logger.info(f"DEBUG: is_standard={is_standard}")
if is_standard:
logger.debug(
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
)
return True
else:
logger.info(
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
)
# For abstract intents, we must visually verify if it actually helped!
# If device is available, we use VLM. If not, we fail safe.
if device:
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
evaluator = SemanticEvaluator()
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
try:
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
decision = _parse_yes_no(response) if response else None
if decision is True:
return True
else:
logger.warning(
f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'. Response: '{response}'"
)
return False
except Exception as e:
logger.error(f"VLM visual verification failed: {e}")
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
return False
# If diff <= 50 for non-toggle
logger.warning(
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
# ── Non-Toggle Structural Delta ──
if diff > 50:
logger.debug(
f"🧠 [ActionMemory] Structural change detected ({diff} chars) for '{intent}'. Verification PASS."
)
return False
return True
logger.warning(
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
)
return False
def _intent_matches_node(intent: str, semantic_string: str) -> bool: