fix(perception): structural Resource-ID bypass gate + dead code purge in ActionMemory
P0-1: Toggle intents (follow/like/save) resolved via Resource-ID fast-path now skip VLM verification entirely. Eliminates the #1 session failure: VLM hallucinating Follow→Like, causing zero follows per session. P0-2: Purge 52 lines of unreachable dead code after unconditional return at line 290. Non-toggle structural delta verification now properly reachable. Remove 4 DEBUG-prefixed logger.info pollution statements. TDD: 7 new tests covering bypass gate, negative guard, non-toggle delta, and debug log pollution detection.
This commit is contained in:
226
tests/core/test_structural_bypass_verification.py
Normal file
226
tests/core/test_structural_bypass_verification.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
P0-1: Structural Bypass Gate — VLM must NEVER be called for Resource-ID matched toggles.
|
||||
P0-2: Dead Code Purge — Non-toggle structural delta must be functional (not dead code).
|
||||
|
||||
TDD RED: These tests encode the exact failures found in session 93c880d3.
|
||||
The VLM hallucinates Follow→Like, causing zero follows in the entire session.
|
||||
The fix: if the clicked element's resource_id structurally matches the intent,
|
||||
skip VLM verification entirely and return True.
|
||||
"""
|
||||
|
||||
from GramAddict.core.perception.action_memory import ActionMemory
|
||||
|
||||
|
||||
class _VLMTrap:
|
||||
"""Device that tracks if VLM was ever attempted. Proves structural bypass.
|
||||
|
||||
We can't rely on raising an exception because the VLM call site catches
|
||||
all exceptions silently. Instead, we track call count and assert on it.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.vlm_call_count = 0
|
||||
|
||||
def get_screenshot_b64(self):
|
||||
self.vlm_call_count += 1
|
||||
return "fake_screenshot_b64"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-1: Structural Bypass for Toggle Intents
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestStructuralBypassFollowButton:
|
||||
"""Session log: VLM classified Follow button as 'Like' 6 times → zero follows."""
|
||||
|
||||
def test_follow_button_with_resource_id_bypasses_vlm(self):
|
||||
"""
|
||||
RED: ActionMemory.verify_success invokes VLM for Follow even when
|
||||
the clicked element has resource_id=profile_header_follow_button.
|
||||
GREEN: When _last_click_context contains a structural resource_id match,
|
||||
VLM verification must be skipped entirely.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap 'Follow' button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/profile_header_follow_button"},
|
||||
"semantic_string": "text: '', desc: 'Follow', id: 'com.instagram.android:id/profile_header_follow_button'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap 'Follow' button",
|
||||
pre_click_xml="<hierarchy><node text='Follow'/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Following'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0, # Low confidence would normally trigger VLM
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
def test_like_button_with_resource_id_bypasses_vlm(self):
|
||||
"""Like button with row_feed_button_like resource_id must skip VLM."""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap like button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_like"},
|
||||
"semantic_string": "text: '', desc: 'Like', id: 'com.instagram.android:id/row_feed_button_like'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap like button",
|
||||
pre_click_xml="<hierarchy><node/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Liked'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
def test_save_button_with_resource_id_bypasses_vlm(self):
|
||||
"""Save/bookmark button with resource_id must skip VLM."""
|
||||
memory = ActionMemory()
|
||||
trap = _VLMTrap()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap save button",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_save"},
|
||||
"semantic_string": "text: '', desc: 'Save', id: 'com.instagram.android:id/row_feed_button_save'",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap save button",
|
||||
pre_click_xml="<hierarchy><node/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Saved'/></hierarchy>",
|
||||
device=trap,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert (
|
||||
trap.vlm_call_count == 0
|
||||
), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!"
|
||||
|
||||
|
||||
class TestStructuralBypassDoesNotApplyToGenericNodes:
|
||||
"""Ensure the bypass only fires for structural Resource-ID matches, not generic nodes."""
|
||||
|
||||
def test_follow_without_resource_id_does_not_bypass(self):
|
||||
"""
|
||||
If the clicked node has no matching resource_id (e.g. VLM-resolved via text),
|
||||
VLM verification must still proceed. We verify by checking that the method
|
||||
falls through to the structural delta path (no VLM device needed = None).
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap 'Follow' button",
|
||||
"node_dict": {"resource_id": ""},
|
||||
"semantic_string": "text: 'Follow', desc: '', id: ''",
|
||||
"xml_context": "<hierarchy><node/></hierarchy>",
|
||||
}
|
||||
|
||||
# With device=None, VLM won't be called, but the structural delta path runs
|
||||
result = memory.verify_success(
|
||||
intent="tap 'Follow' button",
|
||||
pre_click_xml="<hierarchy><node text='Follow'/></hierarchy>",
|
||||
post_click_xml="<hierarchy><node text='Following'/></hierarchy>",
|
||||
device=None,
|
||||
confidence=0.0,
|
||||
)
|
||||
|
||||
# Should fall through to structural delta logic (toggle with diff > 0)
|
||||
assert result is True
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-2: Dead Code Purge — Non-toggle delta must work
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestNonToggleDeltaVerification:
|
||||
"""Lines 291-342 were dead code. Non-toggle structural delta must actually execute."""
|
||||
|
||||
def test_non_toggle_large_delta_passes(self):
|
||||
"""
|
||||
A non-toggle intent (e.g. 'tap explore tab') with a large structural
|
||||
delta (>50 chars diff) should pass verification via the structural
|
||||
delta path, not be silently dropped as dead code.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap explore tab",
|
||||
"node_dict": {"resource_id": "com.instagram.android:id/explore_tab"},
|
||||
"semantic_string": "text: '', desc: 'Explore', id: 'explore_tab'",
|
||||
"xml_context": "",
|
||||
}
|
||||
|
||||
pre_xml = "<hierarchy><node text='Home'/></hierarchy>"
|
||||
post_xml = "<hierarchy>" + "<node text='Explore Item'/>" * 20 + "</hierarchy>"
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap explore tab",
|
||||
pre_click_xml=pre_xml,
|
||||
post_click_xml=post_xml,
|
||||
device=None,
|
||||
confidence=1.0, # High confidence skips VLM
|
||||
)
|
||||
|
||||
# Must not be None — the non-toggle path must be reachable
|
||||
assert result is not None
|
||||
|
||||
def test_non_toggle_zero_delta_fails(self):
|
||||
"""
|
||||
A non-toggle intent with zero structural change should fail verification.
|
||||
This was unreachable dead code before the fix.
|
||||
"""
|
||||
memory = ActionMemory()
|
||||
memory._last_click_context = {
|
||||
"intent": "tap explore tab",
|
||||
"node_dict": {"resource_id": ""},
|
||||
"semantic_string": "text: '', desc: 'Explore', id: ''",
|
||||
"xml_context": "",
|
||||
}
|
||||
|
||||
same_xml = "<hierarchy><node text='Same'/></hierarchy>"
|
||||
|
||||
result = memory.verify_success(
|
||||
intent="tap explore tab",
|
||||
pre_click_xml=same_xml,
|
||||
post_click_xml=same_xml,
|
||||
device=None,
|
||||
confidence=1.0,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# P0-2: Debug Log Pollution Guard
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestNoDebugLogPollution:
|
||||
"""Ensure no 'DEBUG:' prefixed logger.info calls exist in production code."""
|
||||
|
||||
def test_action_memory_has_no_debug_info_logs(self):
|
||||
import inspect
|
||||
|
||||
from GramAddict.core.perception import action_memory
|
||||
|
||||
source = inspect.getsource(action_memory)
|
||||
violations = []
|
||||
for i, line in enumerate(source.splitlines(), 1):
|
||||
if 'logger.info(f"DEBUG:' in line or 'logger.info("DEBUG:' in line:
|
||||
violations.append(f" Line {i}: {line.strip()}")
|
||||
|
||||
assert not violations, "Production code contains DEBUG-prefixed logger.info calls:\n" + "\n".join(violations)
|
||||
Reference in New Issue
Block a user