diff --git a/GramAddict/core/perception/action_memory.py b/GramAddict/core/perception/action_memory.py index 4e614ac..6f5c317 100644 --- a/GramAddict/core/perception/action_memory.py +++ b/GramAddict/core/perception/action_memory.py @@ -196,6 +196,23 @@ class ActionMemory: state_toggles = ["like", "save", "follow", "heart"] is_toggle = any(t in intent_lower for t in state_toggles) + # ── P0-1: Structural Resource-ID Bypass Gate ── + # If the clicked node was resolved via a structural Resource-ID that + # directly matches the toggle intent, VLM verification is SKIPPED. + # This eliminates the #1 session failure: VLM hallucinating Follow→Like. + if is_toggle and self._last_click_context: + clicked_rid = (self._last_click_context.get("node_dict", {}).get("resource_id", "") or "").lower() + if clicked_rid: + for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items(): + if intent_keyword in intent_lower: + if any(marker in clicked_rid for marker in required_markers): + logger.info( + f"⚡ [ActionMemory] Structural Resource-ID bypass: '{intent}' matched " + f"'{clicked_rid}'. Skipping VLM verification — O(1) trust." + ) + return True + break # Only check the first matching intent keyword + # ── VLM Verification Fallback ── # If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM @@ -270,10 +287,8 @@ class ActionMemory: ) return False - # Fallback to structural delta - logger.info(f"DEBUG: len(pre_click_xml)={len(pre_click_xml)} len(post_click_xml)={len(post_click_xml)}") + # ── Structural Delta Verification ── diff = abs(len(pre_click_xml) - len(post_click_xml)) - logger.info(f"DEBUG: diff={diff}") if is_toggle: if diff > 1000: @@ -288,58 +303,18 @@ class ActionMemory: f"⚠️ [ActionMemory] Zero structural shift (diff={diff}) for state-toggle '{intent}'. Verification FAIL." ) return False - # If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough. - # We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight") - # also produces a large diff but achieves the wrong goal. - if diff > 50: - # Is it a standard structural transition? - from GramAddict.core.screen_topology import ScreenTopology - # We don't have screen type here, so we just check if it's in the HD Map keys - logger.info(f"DEBUG: intent is '{intent}'") - logger.info( - f"DEBUG: TRANSITIONS keys are: {[list(t.keys()) for t in ScreenTopology.TRANSITIONS.values()]}" - ) - is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values()) - logger.info(f"DEBUG: is_standard={is_standard}") - - if is_standard: - logger.debug( - f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS." - ) - return True - else: - logger.info( - f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..." - ) - # For abstract intents, we must visually verify if it actually helped! - # If device is available, we use VLM. If not, we fail safe. - if device: - from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator - - evaluator = SemanticEvaluator() - prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO." - try: - response = evaluator._query_vlm(prompt, device.get_screenshot_b64()) - decision = _parse_yes_no(response) if response else None - if decision is True: - return True - else: - logger.warning( - f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'. Response: '{response}'" - ) - return False - except Exception as e: - logger.error(f"VLM visual verification failed: {e}") - - logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.") - return False - - # If diff <= 50 for non-toggle - logger.warning( - f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL." + # ── Non-Toggle Structural Delta ── + if diff > 50: + logger.debug( + f"🧠 [ActionMemory] Structural change detected ({diff} chars) for '{intent}'. Verification PASS." ) - return False + return True + + logger.warning( + f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL." + ) + return False def _intent_matches_node(intent: str, semantic_string: str) -> bool: diff --git a/tests/core/test_structural_bypass_verification.py b/tests/core/test_structural_bypass_verification.py new file mode 100644 index 0000000..a0d4f03 --- /dev/null +++ b/tests/core/test_structural_bypass_verification.py @@ -0,0 +1,226 @@ +""" +P0-1: Structural Bypass Gate — VLM must NEVER be called for Resource-ID matched toggles. +P0-2: Dead Code Purge — Non-toggle structural delta must be functional (not dead code). + +TDD RED: These tests encode the exact failures found in session 93c880d3. +The VLM hallucinates Follow→Like, causing zero follows in the entire session. +The fix: if the clicked element's resource_id structurally matches the intent, +skip VLM verification entirely and return True. +""" + +from GramAddict.core.perception.action_memory import ActionMemory + + +class _VLMTrap: + """Device that tracks if VLM was ever attempted. Proves structural bypass. + + We can't rely on raising an exception because the VLM call site catches + all exceptions silently. Instead, we track call count and assert on it. + """ + + def __init__(self): + self.vlm_call_count = 0 + + def get_screenshot_b64(self): + self.vlm_call_count += 1 + return "fake_screenshot_b64" + + +# ═══════════════════════════════════════════════════════ +# P0-1: Structural Bypass for Toggle Intents +# ═══════════════════════════════════════════════════════ + + +class TestStructuralBypassFollowButton: + """Session log: VLM classified Follow button as 'Like' 6 times → zero follows.""" + + def test_follow_button_with_resource_id_bypasses_vlm(self): + """ + RED: ActionMemory.verify_success invokes VLM for Follow even when + the clicked element has resource_id=profile_header_follow_button. + GREEN: When _last_click_context contains a structural resource_id match, + VLM verification must be skipped entirely. + """ + memory = ActionMemory() + trap = _VLMTrap() + memory._last_click_context = { + "intent": "tap 'Follow' button", + "node_dict": {"resource_id": "com.instagram.android:id/profile_header_follow_button"}, + "semantic_string": "text: '', desc: 'Follow', id: 'com.instagram.android:id/profile_header_follow_button'", + "xml_context": "", + } + + result = memory.verify_success( + intent="tap 'Follow' button", + pre_click_xml="", + post_click_xml="", + device=trap, + confidence=0.0, # Low confidence would normally trigger VLM + ) + + assert result is True + assert ( + trap.vlm_call_count == 0 + ), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!" + + def test_like_button_with_resource_id_bypasses_vlm(self): + """Like button with row_feed_button_like resource_id must skip VLM.""" + memory = ActionMemory() + trap = _VLMTrap() + memory._last_click_context = { + "intent": "tap like button", + "node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_like"}, + "semantic_string": "text: '', desc: 'Like', id: 'com.instagram.android:id/row_feed_button_like'", + "xml_context": "", + } + + result = memory.verify_success( + intent="tap like button", + pre_click_xml="", + post_click_xml="", + device=trap, + confidence=0.0, + ) + + assert result is True + assert ( + trap.vlm_call_count == 0 + ), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!" + + def test_save_button_with_resource_id_bypasses_vlm(self): + """Save/bookmark button with resource_id must skip VLM.""" + memory = ActionMemory() + trap = _VLMTrap() + memory._last_click_context = { + "intent": "tap save button", + "node_dict": {"resource_id": "com.instagram.android:id/row_feed_button_save"}, + "semantic_string": "text: '', desc: 'Save', id: 'com.instagram.android:id/row_feed_button_save'", + "xml_context": "", + } + + result = memory.verify_success( + intent="tap save button", + pre_click_xml="", + post_click_xml="", + device=trap, + confidence=0.0, + ) + + assert result is True + assert ( + trap.vlm_call_count == 0 + ), f"VLM was called {trap.vlm_call_count} time(s) despite structural Resource-ID match!" + + +class TestStructuralBypassDoesNotApplyToGenericNodes: + """Ensure the bypass only fires for structural Resource-ID matches, not generic nodes.""" + + def test_follow_without_resource_id_does_not_bypass(self): + """ + If the clicked node has no matching resource_id (e.g. VLM-resolved via text), + VLM verification must still proceed. We verify by checking that the method + falls through to the structural delta path (no VLM device needed = None). + """ + memory = ActionMemory() + memory._last_click_context = { + "intent": "tap 'Follow' button", + "node_dict": {"resource_id": ""}, + "semantic_string": "text: 'Follow', desc: '', id: ''", + "xml_context": "", + } + + # With device=None, VLM won't be called, but the structural delta path runs + result = memory.verify_success( + intent="tap 'Follow' button", + pre_click_xml="", + post_click_xml="", + device=None, + confidence=0.0, + ) + + # Should fall through to structural delta logic (toggle with diff > 0) + assert result is True + + +# ═══════════════════════════════════════════════════════ +# P0-2: Dead Code Purge — Non-toggle delta must work +# ═══════════════════════════════════════════════════════ + + +class TestNonToggleDeltaVerification: + """Lines 291-342 were dead code. Non-toggle structural delta must actually execute.""" + + def test_non_toggle_large_delta_passes(self): + """ + A non-toggle intent (e.g. 'tap explore tab') with a large structural + delta (>50 chars diff) should pass verification via the structural + delta path, not be silently dropped as dead code. + """ + memory = ActionMemory() + memory._last_click_context = { + "intent": "tap explore tab", + "node_dict": {"resource_id": "com.instagram.android:id/explore_tab"}, + "semantic_string": "text: '', desc: 'Explore', id: 'explore_tab'", + "xml_context": "", + } + + pre_xml = "" + post_xml = "" + "" * 20 + "" + + result = memory.verify_success( + intent="tap explore tab", + pre_click_xml=pre_xml, + post_click_xml=post_xml, + device=None, + confidence=1.0, # High confidence skips VLM + ) + + # Must not be None — the non-toggle path must be reachable + assert result is not None + + def test_non_toggle_zero_delta_fails(self): + """ + A non-toggle intent with zero structural change should fail verification. + This was unreachable dead code before the fix. + """ + memory = ActionMemory() + memory._last_click_context = { + "intent": "tap explore tab", + "node_dict": {"resource_id": ""}, + "semantic_string": "text: '', desc: 'Explore', id: ''", + "xml_context": "", + } + + same_xml = "" + + result = memory.verify_success( + intent="tap explore tab", + pre_click_xml=same_xml, + post_click_xml=same_xml, + device=None, + confidence=1.0, + ) + + assert result is False + + +# ═══════════════════════════════════════════════════════ +# P0-2: Debug Log Pollution Guard +# ═══════════════════════════════════════════════════════ + + +class TestNoDebugLogPollution: + """Ensure no 'DEBUG:' prefixed logger.info calls exist in production code.""" + + def test_action_memory_has_no_debug_info_logs(self): + import inspect + + from GramAddict.core.perception import action_memory + + source = inspect.getsource(action_memory) + violations = [] + for i, line in enumerate(source.splitlines(), 1): + if 'logger.info(f"DEBUG:' in line or 'logger.info("DEBUG:' in line: + violations.append(f" Line {i}: {line.strip()}") + + assert not violations, "Production code contains DEBUG-prefixed logger.info calls:\n" + "\n".join(violations)