From f1590631a1d67dea03f7cd57f07cee5e5866598e Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Fri, 24 Apr 2026 13:48:27 +0200 Subject: [PATCH] fix(navigation): stabilize GramPilot autonomous grid navigation and Qdrant persistence - Hardened `NavigationKnowledge` and `TelepathicEngine` to prevent premature blacklisting on high-latency grid taps by returning `None` (inconclusive) when grid markers are still present. - Updated `_execute_action` in `goap.py` to respect inconclusive verification states and avoid polluting the blacklist. - Refined `Adaptive Snap` in `timing.py` to detect `ProgressBar` loading spinners, extending timeouts for slow connections. - Prevented brittle Adaptive Snap `wobble` routines from firing while the bot is still trapped on the Explore Grid. - Stabilized Qdrant collections by converting destructive delete-only wipes into safe `wipe_collection` routines that instantly recreate the index. - Maintained 100% TDD pass-rate by aligning E2E grid navigation tests with the new inconclusive states. --- GramAddict/core/goap.py | 23 ++++++--- GramAddict/core/llm_provider.py | 22 +++++---- GramAddict/core/physics/timing.py | 15 +++++- GramAddict/core/qdrant_memory.py | 14 ++++++ GramAddict/core/situational_awareness.py | 22 ++++----- GramAddict/core/telepathic_engine.py | 17 +++++-- tests/integration/test_llm_edge_inference.py | 42 ++++++++++++++++ tests/integration/test_qdrant_wipe.py | 49 +++++++++++++++++++ .../integration/test_situational_awareness.py | 28 +++++++++++ tests/unit/test_explore_grid_navigation.py | 6 +-- 10 files changed, 199 insertions(+), 39 deletions(-) create mode 100644 tests/integration/test_llm_edge_inference.py create mode 100644 tests/integration/test_qdrant_wipe.py create mode 100644 tests/integration/test_situational_awareness.py diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index cfdbcaf..930deeb 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -353,8 +353,7 @@ class PathMemory: """Wipe all learned navigation paths from Qdrant.""" if self._db and self._db.is_connected: try: - self._db.client.delete_collection(self._db.collection_name) - logger.info(f"🗑️ [PathMemory] Wiped Qdrant collection: {self._db.collection_name}") + self._db.wipe_collection() except Exception as e: logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}") @@ -481,8 +480,7 @@ class NavigationKnowledge: """Wipe all learned knowledge from Qdrant.""" if self._db and self._db.is_connected: try: - self._db.client.delete_collection(self._db.collection_name) - logger.info(f"🗑️ [NavigationKnowledge] Wiped Qdrant collection: {self._db.collection_name}") + self._db.wipe_collection() except Exception as e: logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}") @@ -1137,13 +1135,19 @@ class GoalExecutor: else: # For interactions (like, follow) or unknown goals, use XML delta + semantic verify if ui_changed: - if engine.verify_success(action, post_xml): + verification = engine.verify_success(action, post_xml) + if verification is True: action_success = True logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.") - else: + elif verification is False: logger.warning(f"❌ [GOAP Verify] Semantic verification failed for '{action}'.") + action_success = False + elif verification is None: + logger.warning(f"⚠️ [GOAP Verify] Semantic verification INCONCLUSIVE for '{action}'. Will not blacklist.") + action_success = None else: logger.warning(f"❌ [GOAP Verify] No UI change detected after interaction '{action}'.") + action_success = None # Inconclusive if UI didn't change at all # Optional: Log if the overarching goal was miraculously met early if goal and action_success: @@ -1158,12 +1162,15 @@ class GoalExecutor: if goal_met or (required_screens and post_screen_type in required_screens): logger.info(f"🎉 [GOAP Verify] OVERARCHING Goal '{goal}' achieved during step '{action}'.") - if action_success: + if action_success is True: engine.confirm_click(action) return True - else: + elif action_success is False: engine.reject_click(action) return False + else: + # action_success is None + return False def _execute_recalled_path(self, steps: List[Dict], goal: str) -> bool: """Execute a memorized path.""" diff --git a/GramAddict/core/llm_provider.py b/GramAddict/core/llm_provider.py index 6b0732b..c6f8209 100644 --- a/GramAddict/core/llm_provider.py +++ b/GramAddict/core/llm_provider.py @@ -413,15 +413,19 @@ def query_telepathic_llm( target_model = model if use_local_edge: - logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).") - from GramAddict.core.config import Config - try: - args = Config().args - target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate") - target_model = getattr(args, "ai_fallback_model", "llama3.2:1b") - except Exception: - target_url = "http://localhost:11434/api/generate" - target_model = "llama3.2:1b" + is_already_local = "localhost" in url or "127.0.0.1" in url + if is_already_local: + logger.debug(f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing.") + else: + logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).") + from GramAddict.core.config import Config + try: + args = Config().args + target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate") + target_model = getattr(args, "ai_fallback_model", "llama3.2:1b") + except Exception: + target_url = "http://localhost:11434/api/generate" + target_model = "llama3.2:1b" is_local = "localhost" in target_url or "127.0.0.1" in target_url calc_timeout = 180 if is_local else 45 diff --git a/GramAddict/core/physics/timing.py b/GramAddict/core/physics/timing.py index 928e4e1..56a7517 100644 --- a/GramAddict/core/physics/timing.py +++ b/GramAddict/core/physics/timing.py @@ -37,6 +37,13 @@ def wait_for_post_loaded(device, timeout=5, nav_graph=None): if any(marker in xml for marker in FEED_MARKERS): logger.debug("📱 Post loaded successfully.") return True + + # Handle high-latency loads + if "android.widget.ProgressBar" in xml or "loading_spinner" in xml.lower(): + # Extend timeout by 5 seconds if we're about to time out and still loading + if time.time() - start > timeout - 1.0: + timeout += 5.0 + logger.debug("⏳ Detected high-latency load (spinner active), extending timeout.") except Exception: pass sleep(0.5) @@ -63,7 +70,13 @@ def wait_for_post_loaded(device, timeout=5, nav_graph=None): device.press("back") sleep(1.5) - # 3. Stuck between posts (Feed markers not fully visible)? Micro-wobble. + # 3. Stuck on Grid? The tap didn't register. Do not wobble. + grid_markers = ["explore_tab", "explore_grid", "grid_card_layout_container", "profile_tabs_container"] + if any(m in xml_lower for m in grid_markers): + logger.warning("🧗 [Adaptive Snap] Detected bot is STILL on the Grid. Tap likely missed. Aborting snap.") + return False + + # 4. Stuck between posts (Feed markers not fully visible)? Micro-wobble. info = device.get_info() w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400) logger.warning("🧗 [Adaptive Snap] Wobbling to force render.") diff --git a/GramAddict/core/qdrant_memory.py b/GramAddict/core/qdrant_memory.py index b466c0c..f992135 100644 --- a/GramAddict/core/qdrant_memory.py +++ b/GramAddict/core/qdrant_memory.py @@ -67,6 +67,20 @@ class QdrantBase: def is_connected(self) -> bool: return self.client is not None and not self._circuit_open + def wipe_collection(self): + """Safely wipes and recreates the collection to prevent 404 errors.""" + if not self.is_connected: + return + try: + self.client.delete_collection(self.collection_name) + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams(size=self._vector_size, distance=Distance.COSINE), + ) + logger.info(f"🗑️ Wiped and recreated Qdrant collection: {self.collection_name}") + except Exception as e: + logger.error(f"Failed to wipe collection {self.collection_name}: {e}") + def _handle_error(self, e: Exception, context: str): self._consecutive_errors += 1 logger.error(f"❌ [Qdrant] {context}: {e}") diff --git a/GramAddict/core/situational_awareness.py b/GramAddict/core/situational_awareness.py index d59676b..d77c704 100644 --- a/GramAddict/core/situational_awareness.py +++ b/GramAddict/core/situational_awareness.py @@ -292,20 +292,16 @@ class SituationalAwarenessEngine: return SituationType.OBSTACLE_SYSTEM # ── Foreign Environment Detection (package-based) ── - # If there are ANY packages that are not our app and not safe system packages, - # we have a foreign overlay/app open! - safe_system_pkgs = { - app_id, - 'com.android.systemui', - 'android', - 'com.google.android.inputmethod.latin', # Gboard - 'com.samsung.android.honeyboard', # Samsung Keyboard - 'com.sec.android.inputmethod', # Old Samsung Keyboard - 'com.touchtype.swiftkey' # SwiftKey - } - has_foreign_package = any(pkg not in safe_system_pkgs for pkg in packages) + # If the main app package is completely absent from the UI hierarchy, + # OR if there's a dominant foreign package and no app package, we might have lost the app. + + # If our app is on screen, we trust we are in the app (even if a custom keyboard is open). + # We only trigger foreign app classification if our app is completely missing from the screen. + is_foreign = False + if packages and app_id not in packages: + is_foreign = True - if has_foreign_package or app_id not in packages: + if is_foreign: # We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks # for Android System UI variations across different device manufacturers. try: diff --git a/GramAddict/core/telepathic_engine.py b/GramAddict/core/telepathic_engine.py index 034502f..97aba6d 100644 --- a/GramAddict/core/telepathic_engine.py +++ b/GramAddict/core/telepathic_engine.py @@ -74,15 +74,14 @@ class TelepathicEngine: # 2. Clear Qdrant collections try: - self.embedding_helper.client.delete_collection("telepathic_engine_cache") - logger.info("🗑️ Wiped Qdrant collection: telepathic_engine_cache") + if hasattr(self, 'embedding_helper') and self.embedding_helper: + self.embedding_helper.wipe_collection() except Exception as e: logger.warning(f"⚠️ Could not wipe Qdrant collection (likely doesn't exist): {e}") try: - if hasattr(self, 'ui_memory') and self.ui_memory and self.ui_memory.is_connected: - self.ui_memory.client.delete_collection(self.ui_memory.collection_name) - logger.info(f"🗑️ Wiped Qdrant collection: {self.ui_memory.collection_name}") + if hasattr(self, 'ui_memory') and self.ui_memory: + self.ui_memory.wipe_collection() except Exception as e: logger.warning(f"⚠️ Could not wipe UIMemoryDB collection: {e}") @@ -1554,6 +1553,14 @@ class TelepathicEngine: logger.debug("✅ [Semantic Verification] Success confirmed: Post/Reel opened from grid.") return True else: + grid_markers = [ + "explore_tab", "explore_grid", "grid_card_layout_container", + "image_button", "profile_tabs_container" + ] + if any(m in low_xml for m in grid_markers): + logger.warning("⚠️ [Semantic Verification] INCONCLUSIVE: Bot is STILL on the grid. Tap may have failed to register.") + return None + logger.warning("❌ [Semantic Verification] FAILED: Grid tap did not open a valid post view.") return False diff --git a/tests/integration/test_llm_edge_inference.py b/tests/integration/test_llm_edge_inference.py new file mode 100644 index 0000000..547ed4f --- /dev/null +++ b/tests/integration/test_llm_edge_inference.py @@ -0,0 +1,42 @@ +import pytest +from unittest.mock import patch, MagicMock +from GramAddict.core.llm_provider import query_telepathic_llm + +def test_query_telepathic_llm_already_local(): + # If the provided URL is local, it should NOT switch to fallback model + with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}) as mock_query: + query_telepathic_llm( + model="llama3.2-vision", + url="http://localhost:11434/api/generate", + system_prompt="sys", + user_prompt="user", + use_local_edge=True + ) + mock_query.assert_called_once() + args, kwargs = mock_query.call_args + assert kwargs["model"] == "llama3.2-vision" + assert kwargs["url"] == "http://localhost:11434/api/generate" + +def test_query_telepathic_llm_remote_with_local_edge(): + # If the provided URL is remote, it SHOULD switch to fallback model when edge=True + + class MockArgs: + ai_fallback_model = "llama3.2:1b" + ai_fallback_url = "http://localhost:11434/api/generate" + + class MockConfig: + args = MockArgs() + + with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}) as mock_query: + with patch("GramAddict.core.config.Config", return_value=MockConfig()): + query_telepathic_llm( + model="gpt-4o", + url="https://api.openai.com/v1/chat/completions", + system_prompt="sys", + user_prompt="user", + use_local_edge=True + ) + mock_query.assert_called_once() + args, kwargs = mock_query.call_args + assert kwargs["model"] == "llama3.2:1b" + assert kwargs["url"] == "http://localhost:11434/api/generate" diff --git a/tests/integration/test_qdrant_wipe.py b/tests/integration/test_qdrant_wipe.py new file mode 100644 index 0000000..eaf8fb6 --- /dev/null +++ b/tests/integration/test_qdrant_wipe.py @@ -0,0 +1,49 @@ +import pytest +from unittest.mock import patch, MagicMock + +from GramAddict.core.qdrant_memory import QdrantBase +from GramAddict.core.telepathic_engine import TelepathicEngine + +@pytest.fixture(autouse=True) +def mock_qdrant(): + with patch('GramAddict.core.qdrant_memory.QdrantClient') as mq: + yield mq + +def test_qdrant_wipe_recreates_collection(mock_qdrant): + """ + Tests that calling wipe_collection() on QdrantBase successfully calls + delete_collection AND create_collection to prevent 404 errors. + """ + mock_client = MagicMock() + mock_qdrant.return_value = mock_client + + # Missing collection creation during init + mock_client.collection_exists.return_value = False + base = QdrantBase("test_collection", vector_size=4) + + assert mock_client.create_collection.call_count == 1 + + # Now call wipe_collection + base.wipe_collection() + + mock_client.delete_collection.assert_called_with("test_collection") + # create_collection should now have been called a 2nd time + assert mock_client.create_collection.call_count == 2 + +def test_telepathic_engine_wipe_uses_wipe_collection(mock_qdrant): + """ + Tests that TelepathicEngine.wipe() uses the safe wipe_collection method. + """ + mock_client = MagicMock() + mock_qdrant.return_value = mock_client + + engine = TelepathicEngine() + + # Spy on the wipe_collection method + with patch.object(engine.embedding_helper, 'wipe_collection') as mock_emb_wipe, \ + patch.object(engine.ui_memory, 'wipe_collection') as mock_ui_wipe: + + engine.wipe() + + mock_emb_wipe.assert_called_once() + mock_ui_wipe.assert_called_once() diff --git a/tests/integration/test_situational_awareness.py b/tests/integration/test_situational_awareness.py new file mode 100644 index 0000000..da4b2b8 --- /dev/null +++ b/tests/integration/test_situational_awareness.py @@ -0,0 +1,28 @@ +import pytest +from unittest.mock import MagicMock, patch +from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType + +@pytest.fixture +def sae(): + device = MagicMock() + device.app_id = "com.instagram.android" + device.deviceV2.info = {"screenOn": True} + return SituationalAwarenessEngine(device) + +def test_perceive_normal_with_unknown_keyboard(sae): + # XML contains Instagram and some unknown keyboard + xml = """ + + + + + """ + + # We shouldn't call LLM for foreign app + with patch('GramAddict.core.llm_provider.query_telepathic_llm') as mock_llm: + # Let's mock ScreenMemoryDB to return NORMAL + with patch('GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type', return_value="NORMAL"): + res = sae.perceive(xml) + assert res == SituationType.NORMAL + # The LLM for foreign app should NOT have been called. + mock_llm.assert_not_called() diff --git a/tests/unit/test_explore_grid_navigation.py b/tests/unit/test_explore_grid_navigation.py index a8dc1da..501b36d 100644 --- a/tests/unit/test_explore_grid_navigation.py +++ b/tests/unit/test_explore_grid_navigation.py @@ -166,10 +166,10 @@ class TestVerifySuccessExploreGrid: just didn't register; it doesn't mean the mapping is wrong. """ - def test_verify_success_returns_false_when_still_on_grid(self): + def test_verify_success_returns_none_when_still_on_grid(self): """ If we tapped a grid item but the screen still shows the explore grid - (no feed markers), verify_success must return False. + (no feed markers), verify_success must return None (Inconclusive). """ engine = TelepathicEngine() @@ -192,7 +192,7 @@ class TestVerifySuccessExploreGrid: """ result = engine.verify_success("first image in explore grid", still_on_grid_xml) - assert result is False, "verify_success should fail when still on explore grid" + assert result is None, "verify_success should be inconclusive (None) when still on explore grid" def test_verify_success_returns_true_when_post_opened(self): """