fix(perception): structural Resource-ID bypass gate + dead code purge in ActionMemory
P0-1: Toggle intents (follow/like/save) resolved via Resource-ID fast-path now skip VLM verification entirely. Eliminates the #1 session failure: VLM hallucinating Follow→Like, causing zero follows per session. P0-2: Purge 52 lines of unreachable dead code after unconditional return at line 290. Non-toggle structural delta verification now properly reachable. Remove 4 DEBUG-prefixed logger.info pollution statements. TDD: 7 new tests covering bypass gate, negative guard, non-toggle delta, and debug log pollution detection.
This commit is contained in:
@@ -196,6 +196,23 @@ class ActionMemory:
|
||||
state_toggles = ["like", "save", "follow", "heart"]
|
||||
is_toggle = any(t in intent_lower for t in state_toggles)
|
||||
|
||||
# ── P0-1: Structural Resource-ID Bypass Gate ──
|
||||
# If the clicked node was resolved via a structural Resource-ID that
|
||||
# directly matches the toggle intent, VLM verification is SKIPPED.
|
||||
# This eliminates the #1 session failure: VLM hallucinating Follow→Like.
|
||||
if is_toggle and self._last_click_context:
|
||||
clicked_rid = (self._last_click_context.get("node_dict", {}).get("resource_id", "") or "").lower()
|
||||
if clicked_rid:
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in clicked_rid for marker in required_markers):
|
||||
logger.info(
|
||||
f"⚡ [ActionMemory] Structural Resource-ID bypass: '{intent}' matched "
|
||||
f"'{clicked_rid}'. Skipping VLM verification — O(1) trust."
|
||||
)
|
||||
return True
|
||||
break # Only check the first matching intent keyword
|
||||
|
||||
# ── VLM Verification Fallback ──
|
||||
|
||||
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
|
||||
@@ -270,10 +287,8 @@ class ActionMemory:
|
||||
)
|
||||
return False
|
||||
|
||||
# Fallback to structural delta
|
||||
logger.info(f"DEBUG: len(pre_click_xml)={len(pre_click_xml)} len(post_click_xml)={len(post_click_xml)}")
|
||||
# ── Structural Delta Verification ──
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
logger.info(f"DEBUG: diff={diff}")
|
||||
|
||||
if is_toggle:
|
||||
if diff > 1000:
|
||||
@@ -288,58 +303,18 @@ class ActionMemory:
|
||||
f"⚠️ [ActionMemory] Zero structural shift (diff={diff}) for state-toggle '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
|
||||
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
|
||||
# also produces a large diff but achieves the wrong goal.
|
||||
if diff > 50:
|
||||
# Is it a standard structural transition?
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
# We don't have screen type here, so we just check if it's in the HD Map keys
|
||||
logger.info(f"DEBUG: intent is '{intent}'")
|
||||
logger.info(
|
||||
f"DEBUG: TRANSITIONS keys are: {[list(t.keys()) for t in ScreenTopology.TRANSITIONS.values()]}"
|
||||
)
|
||||
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
|
||||
logger.info(f"DEBUG: is_standard={is_standard}")
|
||||
|
||||
if is_standard:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
|
||||
)
|
||||
# For abstract intents, we must visually verify if it actually helped!
|
||||
# If device is available, we use VLM. If not, we fail safe.
|
||||
if device:
|
||||
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
|
||||
|
||||
evaluator = SemanticEvaluator()
|
||||
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
|
||||
try:
|
||||
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
|
||||
decision = _parse_yes_no(response) if response else None
|
||||
if decision is True:
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'. Response: '{response}'"
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"VLM visual verification failed: {e}")
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
|
||||
return False
|
||||
|
||||
# If diff <= 50 for non-toggle
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
|
||||
# ── Non-Toggle Structural Delta ──
|
||||
if diff > 50:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected ({diff} chars) for '{intent}'. Verification PASS."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
|
||||
226
tests/core/test_structural_bypass_verification.py
Normal file
226
tests/core/test_structural_bypass_verification.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
P0-1: Structural Bypass Gate — VLM must NEVER be called for Resource-ID matched toggles.
|
||||
P0-2: Dead Code Purge — Non-toggle structural delta must be functional (not dead code).
|
||||
|
||||
TDD RED: These tests encode the exact failures found in session 93c880d3.
|
||||
The VLM hallucinates Follow→Like, causing zero follows in the entire session.
|
||||
The fix: if the clicked element's resource_id structurally matches the intent,
|
||||
skip VLM verification entirely and return True.
|
||||
"""
|
||||
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
|
||||
class _VLMTrap:
|
||||
"""Device that tracks if VLM was ever attempted. Proves structural bypass.
|
||||
|
||||
We can't rely on raising an exception because the VLM call site catches
|
||||
all exceptions silently. Instead, we track call count and assert on it.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.vlm_call_count = 0
|
||||
|
||||
def get_screenshot_b64(self):
|
||||
self.vlm_call_count += 1
|
||||
return "fake_screenshot_b64"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-1: Structural Bypass for Toggle Intents
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestStructuralBypassFollowButton:
|
||||
"""Session log: VLM classified Follow button as 'Like' 6 times → zero follows."""
|
||||
|
||||
def test_follow_button_with_resource_id_bypasses_vlm(self):
|
||||
"""
|
||||
RED: ActionMemory.verify_success invokes VLM for Follow even when
|
||||
the clicked element has resource_id=profile_header_follow_button.
|
||||
GREEN: When _last_click_context contains a structural resource_id match,
|
||||
VLM verification must be skipped entirely.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap 'Follow' button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/profile_header_follow_button"},
|
||||
"semantic_string": "text: '', desc: 'Follow', id: 'com.instagram.android:id/profile_header_follow_button'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap 'Follow' button",
|
||||
pre_click_xml="<hierarchy><node text='Follow'/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Following'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0, # Low confidence would normally trigger VLM
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
def test_like_button_with_resource_id_bypasses_vlm(self):
|
||||
"""Like button with row_feed_button_like resource_id must skip VLM."""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap like button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_like"},
|
||||
"semantic_string": "text: '', desc: 'Like', id: 'com.instagram.android:id/row_feed_button_like'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap like button",
|
||||
pre_click_xml="<hierarchy><node/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Liked'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
def test_save_button_with_resource_id_bypasses_vlm(self):
|
||||
"""Save/bookmark button with resource_id must skip VLM."""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap save button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_save"},
|
||||
"semantic_string": "text: '', desc: 'Save', id: 'com.instagram.android:id/row_feed_button_save'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap save button",
|
||||
pre_click_xml="<hierarchy><node/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Saved'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
|
||||
class TestStructuralBypassDoesNotApplyToGenericNodes:
|
||||
"""Ensure the bypass only fires for structural Resource-ID matches, not generic nodes."""
|
||||
|
||||
def test_follow_without_resource_id_does_not_bypass(self):
|
||||
"""
|
||||
If the clicked node has no matching resource_id (e.g. VLM-resolved via text),
|
||||
VLM verification must still proceed. We verify by checking that the method
|
||||
falls through to the structural delta path (no VLM device needed = None).
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap 'Follow' button",
|
||||
"node_dict": {"resource_id": ""},
|
||||
"semantic_string": "text: 'Follow', desc: '', id: ''",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
# With device=None, VLM won't be called, but the structural delta path runs
|
||||
result = memory.verify_success(
|
||||
intent="tap 'Follow' button",
|
||||
pre_click_xml="<hierarchy><node text='Follow'/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Following'/></hierarchy>",
|
||||
device=None,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# Should fall through to structural delta logic (toggle with diff > 0)
|
||||
assert result is True
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-2: Dead Code Purge — Non-toggle delta must work
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestNonToggleDeltaVerification:
|
||||
"""Lines 291-342 were dead code. Non-toggle structural delta must actually execute."""
|
||||
|
||||
def test_non_toggle_large_delta_passes(self):
|
||||
"""
|
||||
A non-toggle intent (e.g. 'tap explore tab') with a large structural
|
||||
delta (>50 chars diff) should pass verification via the structural
|
||||
delta path, not be silently dropped as dead code.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap explore tab",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/explore_tab"},
|
||||
"semantic_string": "text: '', desc: 'Explore', id: 'explore_tab'",
|
||||
"xml_context": "",
|
||||
}
|
||||
|
||||
pre_xml = "<hierarchy><node text='Home'/></hierarchy>"
|
||||
post_xml = "<hierarchy>" + "<node text='Explore Item'/>" * 20 + "</hierarchy>"
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap explore tab",
|
||||
pre_click_xml=pre_xml,
|
||||
post_click_xml=post_xml,
|
||||
device=None,
|
||||
confidence=1.0, # High confidence skips VLM
|
||||
)
|
||||
|
||||
# Must not be None — the non-toggle path must be reachable
|
||||
assert result is not None
|
||||
|
||||
def test_non_toggle_zero_delta_fails(self):
|
||||
"""
|
||||
A non-toggle intent with zero structural change should fail verification.
|
||||
This was unreachable dead code before the fix.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap explore tab",
|
||||
"node_dict": {"resource_id": ""},
|
||||
"semantic_string": "text: '', desc: 'Explore', id: ''",
|
||||
"xml_context": "",
|
||||
}
|
||||
|
||||
same_xml = "<hierarchy><node text='Same'/></hierarchy>"
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap explore tab",
|
||||
pre_click_xml=same_xml,
|
||||
post_click_xml=same_xml,
|
||||
device=None,
|
||||
confidence=1.0,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-2: Debug Log Pollution Guard
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestNoDebugLogPollution:
|
||||
"""Ensure no 'DEBUG:' prefixed logger.info calls exist in production code."""
|
||||
|
||||
def test_action_memory_has_no_debug_info_logs(self):
|
||||
import inspect
|
||||
|
||||
from GramAddict.core.perception import action_memory
|
||||
|
||||
source = inspect.getsource(action_memory)
|
||||
violations = []
|
||||
for i, line in enumerate(source.splitlines(), 1):
|
||||
if 'logger.info(f"DEBUG:' in line or 'logger.info("DEBUG:' in line:
|
||||
violations.append(f" Line {i}: {line.strip()}")
|
||||
|
||||
assert not violations, "Production code contains DEBUG-prefixed logger.info calls:\n" + "\n".join(violations)
|
||||
Reference in New Issue
Block a user