fix(navigation): stabilize GramPilot autonomous grid navigation and Qdrant persistence
- Hardened `NavigationKnowledge` and `TelepathicEngine` to prevent premature blacklisting on high-latency grid taps by returning `None` (inconclusive) when grid markers are still present. - Updated `_execute_action` in `goap.py` to respect inconclusive verification states and avoid polluting the blacklist. - Refined `Adaptive Snap` in `timing.py` to detect `ProgressBar` loading spinners, extending timeouts for slow connections. - Prevented brittle Adaptive Snap `wobble` routines from firing while the bot is still trapped on the Explore Grid. - Stabilized Qdrant collections by converting destructive delete-only wipes into safe `wipe_collection` routines that instantly recreate the index. - Maintained 100% TDD pass-rate by aligning E2E grid navigation tests with the new inconclusive states.
This commit is contained in:
@@ -353,8 +353,7 @@ class PathMemory:
|
||||
"""Wipe all learned navigation paths from Qdrant."""
|
||||
if self._db and self._db.is_connected:
|
||||
try:
|
||||
self._db.client.delete_collection(self._db.collection_name)
|
||||
logger.info(f"🗑️ [PathMemory] Wiped Qdrant collection: {self._db.collection_name}")
|
||||
self._db.wipe_collection()
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}")
|
||||
|
||||
@@ -481,8 +480,7 @@ class NavigationKnowledge:
|
||||
"""Wipe all learned knowledge from Qdrant."""
|
||||
if self._db and self._db.is_connected:
|
||||
try:
|
||||
self._db.client.delete_collection(self._db.collection_name)
|
||||
logger.info(f"🗑️ [NavigationKnowledge] Wiped Qdrant collection: {self._db.collection_name}")
|
||||
self._db.wipe_collection()
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}")
|
||||
|
||||
@@ -1137,13 +1135,19 @@ class GoalExecutor:
|
||||
else:
|
||||
# For interactions (like, follow) or unknown goals, use XML delta + semantic verify
|
||||
if ui_changed:
|
||||
if engine.verify_success(action, post_xml):
|
||||
verification = engine.verify_success(action, post_xml)
|
||||
if verification is True:
|
||||
action_success = True
|
||||
logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.")
|
||||
else:
|
||||
elif verification is False:
|
||||
logger.warning(f"❌ [GOAP Verify] Semantic verification failed for '{action}'.")
|
||||
action_success = False
|
||||
elif verification is None:
|
||||
logger.warning(f"⚠️ [GOAP Verify] Semantic verification INCONCLUSIVE for '{action}'. Will not blacklist.")
|
||||
action_success = None
|
||||
else:
|
||||
logger.warning(f"❌ [GOAP Verify] No UI change detected after interaction '{action}'.")
|
||||
action_success = None # Inconclusive if UI didn't change at all
|
||||
|
||||
# Optional: Log if the overarching goal was miraculously met early
|
||||
if goal and action_success:
|
||||
@@ -1158,12 +1162,15 @@ class GoalExecutor:
|
||||
if goal_met or (required_screens and post_screen_type in required_screens):
|
||||
logger.info(f"🎉 [GOAP Verify] OVERARCHING Goal '{goal}' achieved during step '{action}'.")
|
||||
|
||||
if action_success:
|
||||
if action_success is True:
|
||||
engine.confirm_click(action)
|
||||
return True
|
||||
else:
|
||||
elif action_success is False:
|
||||
engine.reject_click(action)
|
||||
return False
|
||||
else:
|
||||
# action_success is None
|
||||
return False
|
||||
|
||||
def _execute_recalled_path(self, steps: List[Dict], goal: str) -> bool:
|
||||
"""Execute a memorized path."""
|
||||
|
||||
@@ -413,15 +413,19 @@ def query_telepathic_llm(
|
||||
target_model = model
|
||||
|
||||
if use_local_edge:
|
||||
logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).")
|
||||
from GramAddict.core.config import Config
|
||||
try:
|
||||
args = Config().args
|
||||
target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
|
||||
target_model = getattr(args, "ai_fallback_model", "llama3.2:1b")
|
||||
except Exception:
|
||||
target_url = "http://localhost:11434/api/generate"
|
||||
target_model = "llama3.2:1b"
|
||||
is_already_local = "localhost" in url or "127.0.0.1" in url
|
||||
if is_already_local:
|
||||
logger.debug(f"⚡ [Edge Inference] Primary model {model} is already local. Using it directly to prevent VRAM thrashing.")
|
||||
else:
|
||||
logger.info("⚡ [Edge Inference] Routing telepathic request to local Ollama host (0ms latency target).")
|
||||
from GramAddict.core.config import Config
|
||||
try:
|
||||
args = Config().args
|
||||
target_url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
|
||||
target_model = getattr(args, "ai_fallback_model", "llama3.2:1b")
|
||||
except Exception:
|
||||
target_url = "http://localhost:11434/api/generate"
|
||||
target_model = "llama3.2:1b"
|
||||
|
||||
is_local = "localhost" in target_url or "127.0.0.1" in target_url
|
||||
calc_timeout = 180 if is_local else 45
|
||||
|
||||
@@ -37,6 +37,13 @@ def wait_for_post_loaded(device, timeout=5, nav_graph=None):
|
||||
if any(marker in xml for marker in FEED_MARKERS):
|
||||
logger.debug("📱 Post loaded successfully.")
|
||||
return True
|
||||
|
||||
# Handle high-latency loads
|
||||
if "android.widget.ProgressBar" in xml or "loading_spinner" in xml.lower():
|
||||
# Extend timeout by 5 seconds if we're about to time out and still loading
|
||||
if time.time() - start > timeout - 1.0:
|
||||
timeout += 5.0
|
||||
logger.debug("⏳ Detected high-latency load (spinner active), extending timeout.")
|
||||
except Exception:
|
||||
pass
|
||||
sleep(0.5)
|
||||
@@ -63,7 +70,13 @@ def wait_for_post_loaded(device, timeout=5, nav_graph=None):
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
# 3. Stuck between posts (Feed markers not fully visible)? Micro-wobble.
|
||||
# 3. Stuck on Grid? The tap didn't register. Do not wobble.
|
||||
grid_markers = ["explore_tab", "explore_grid", "grid_card_layout_container", "profile_tabs_container"]
|
||||
if any(m in xml_lower for m in grid_markers):
|
||||
logger.warning("🧗 [Adaptive Snap] Detected bot is STILL on the Grid. Tap likely missed. Aborting snap.")
|
||||
return False
|
||||
|
||||
# 4. Stuck between posts (Feed markers not fully visible)? Micro-wobble.
|
||||
info = device.get_info()
|
||||
w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400)
|
||||
logger.warning("🧗 [Adaptive Snap] Wobbling to force render.")
|
||||
|
||||
@@ -67,6 +67,20 @@ class QdrantBase:
|
||||
def is_connected(self) -> bool:
|
||||
return self.client is not None and not self._circuit_open
|
||||
|
||||
def wipe_collection(self):
|
||||
"""Safely wipes and recreates the collection to prevent 404 errors."""
|
||||
if not self.is_connected:
|
||||
return
|
||||
try:
|
||||
self.client.delete_collection(self.collection_name)
|
||||
self.client.create_collection(
|
||||
collection_name=self.collection_name,
|
||||
vectors_config=VectorParams(size=self._vector_size, distance=Distance.COSINE),
|
||||
)
|
||||
logger.info(f"🗑️ Wiped and recreated Qdrant collection: {self.collection_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to wipe collection {self.collection_name}: {e}")
|
||||
|
||||
def _handle_error(self, e: Exception, context: str):
|
||||
self._consecutive_errors += 1
|
||||
logger.error(f"❌ [Qdrant] {context}: {e}")
|
||||
|
||||
@@ -292,20 +292,16 @@ class SituationalAwarenessEngine:
|
||||
return SituationType.OBSTACLE_SYSTEM
|
||||
|
||||
# ── Foreign Environment Detection (package-based) ──
|
||||
# If there are ANY packages that are not our app and not safe system packages,
|
||||
# we have a foreign overlay/app open!
|
||||
safe_system_pkgs = {
|
||||
app_id,
|
||||
'com.android.systemui',
|
||||
'android',
|
||||
'com.google.android.inputmethod.latin', # Gboard
|
||||
'com.samsung.android.honeyboard', # Samsung Keyboard
|
||||
'com.sec.android.inputmethod', # Old Samsung Keyboard
|
||||
'com.touchtype.swiftkey' # SwiftKey
|
||||
}
|
||||
has_foreign_package = any(pkg not in safe_system_pkgs for pkg in packages)
|
||||
# If the main app package is completely absent from the UI hierarchy,
|
||||
# OR if there's a dominant foreign package and no app package, we might have lost the app.
|
||||
|
||||
# If our app is on screen, we trust we are in the app (even if a custom keyboard is open).
|
||||
# We only trigger foreign app classification if our app is completely missing from the screen.
|
||||
is_foreign = False
|
||||
if packages and app_id not in packages:
|
||||
is_foreign = True
|
||||
|
||||
if has_foreign_package or app_id not in packages:
|
||||
if is_foreign:
|
||||
# We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks
|
||||
# for Android System UI variations across different device manufacturers.
|
||||
try:
|
||||
|
||||
@@ -74,15 +74,14 @@ class TelepathicEngine:
|
||||
|
||||
# 2. Clear Qdrant collections
|
||||
try:
|
||||
self.embedding_helper.client.delete_collection("telepathic_engine_cache")
|
||||
logger.info("🗑️ Wiped Qdrant collection: telepathic_engine_cache")
|
||||
if hasattr(self, 'embedding_helper') and self.embedding_helper:
|
||||
self.embedding_helper.wipe_collection()
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not wipe Qdrant collection (likely doesn't exist): {e}")
|
||||
|
||||
try:
|
||||
if hasattr(self, 'ui_memory') and self.ui_memory and self.ui_memory.is_connected:
|
||||
self.ui_memory.client.delete_collection(self.ui_memory.collection_name)
|
||||
logger.info(f"🗑️ Wiped Qdrant collection: {self.ui_memory.collection_name}")
|
||||
if hasattr(self, 'ui_memory') and self.ui_memory:
|
||||
self.ui_memory.wipe_collection()
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not wipe UIMemoryDB collection: {e}")
|
||||
|
||||
@@ -1554,6 +1553,14 @@ class TelepathicEngine:
|
||||
logger.debug("✅ [Semantic Verification] Success confirmed: Post/Reel opened from grid.")
|
||||
return True
|
||||
else:
|
||||
grid_markers = [
|
||||
"explore_tab", "explore_grid", "grid_card_layout_container",
|
||||
"image_button", "profile_tabs_container"
|
||||
]
|
||||
if any(m in low_xml for m in grid_markers):
|
||||
logger.warning("⚠️ [Semantic Verification] INCONCLUSIVE: Bot is STILL on the grid. Tap may have failed to register.")
|
||||
return None
|
||||
|
||||
logger.warning("❌ [Semantic Verification] FAILED: Grid tap did not open a valid post view.")
|
||||
return False
|
||||
|
||||
|
||||
42
tests/integration/test_llm_edge_inference.py
Normal file
42
tests/integration/test_llm_edge_inference.py
Normal file
@@ -0,0 +1,42 @@
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
def test_query_telepathic_llm_already_local():
|
||||
# If the provided URL is local, it should NOT switch to fallback model
|
||||
with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}) as mock_query:
|
||||
query_telepathic_llm(
|
||||
model="llama3.2-vision",
|
||||
url="http://localhost:11434/api/generate",
|
||||
system_prompt="sys",
|
||||
user_prompt="user",
|
||||
use_local_edge=True
|
||||
)
|
||||
mock_query.assert_called_once()
|
||||
args, kwargs = mock_query.call_args
|
||||
assert kwargs["model"] == "llama3.2-vision"
|
||||
assert kwargs["url"] == "http://localhost:11434/api/generate"
|
||||
|
||||
def test_query_telepathic_llm_remote_with_local_edge():
|
||||
# If the provided URL is remote, it SHOULD switch to fallback model when edge=True
|
||||
|
||||
class MockArgs:
|
||||
ai_fallback_model = "llama3.2:1b"
|
||||
ai_fallback_url = "http://localhost:11434/api/generate"
|
||||
|
||||
class MockConfig:
|
||||
args = MockArgs()
|
||||
|
||||
with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}) as mock_query:
|
||||
with patch("GramAddict.core.config.Config", return_value=MockConfig()):
|
||||
query_telepathic_llm(
|
||||
model="gpt-4o",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
system_prompt="sys",
|
||||
user_prompt="user",
|
||||
use_local_edge=True
|
||||
)
|
||||
mock_query.assert_called_once()
|
||||
args, kwargs = mock_query.call_args
|
||||
assert kwargs["model"] == "llama3.2:1b"
|
||||
assert kwargs["url"] == "http://localhost:11434/api/generate"
|
||||
49
tests/integration/test_qdrant_wipe.py
Normal file
49
tests/integration/test_qdrant_wipe.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from GramAddict.core.qdrant_memory import QdrantBase
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_qdrant():
|
||||
with patch('GramAddict.core.qdrant_memory.QdrantClient') as mq:
|
||||
yield mq
|
||||
|
||||
def test_qdrant_wipe_recreates_collection(mock_qdrant):
|
||||
"""
|
||||
Tests that calling wipe_collection() on QdrantBase successfully calls
|
||||
delete_collection AND create_collection to prevent 404 errors.
|
||||
"""
|
||||
mock_client = MagicMock()
|
||||
mock_qdrant.return_value = mock_client
|
||||
|
||||
# Missing collection creation during init
|
||||
mock_client.collection_exists.return_value = False
|
||||
base = QdrantBase("test_collection", vector_size=4)
|
||||
|
||||
assert mock_client.create_collection.call_count == 1
|
||||
|
||||
# Now call wipe_collection
|
||||
base.wipe_collection()
|
||||
|
||||
mock_client.delete_collection.assert_called_with("test_collection")
|
||||
# create_collection should now have been called a 2nd time
|
||||
assert mock_client.create_collection.call_count == 2
|
||||
|
||||
def test_telepathic_engine_wipe_uses_wipe_collection(mock_qdrant):
|
||||
"""
|
||||
Tests that TelepathicEngine.wipe() uses the safe wipe_collection method.
|
||||
"""
|
||||
mock_client = MagicMock()
|
||||
mock_qdrant.return_value = mock_client
|
||||
|
||||
engine = TelepathicEngine()
|
||||
|
||||
# Spy on the wipe_collection method
|
||||
with patch.object(engine.embedding_helper, 'wipe_collection') as mock_emb_wipe, \
|
||||
patch.object(engine.ui_memory, 'wipe_collection') as mock_ui_wipe:
|
||||
|
||||
engine.wipe()
|
||||
|
||||
mock_emb_wipe.assert_called_once()
|
||||
mock_ui_wipe.assert_called_once()
|
||||
28
tests/integration/test_situational_awareness.py
Normal file
28
tests/integration/test_situational_awareness.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
|
||||
@pytest.fixture
|
||||
def sae():
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device.deviceV2.info = {"screenOn": True}
|
||||
return SituationalAwarenessEngine(device)
|
||||
|
||||
def test_perceive_normal_with_unknown_keyboard(sae):
|
||||
# XML contains Instagram and some unknown keyboard
|
||||
xml = """
|
||||
<hierarchy>
|
||||
<node package="com.instagram.android" resource-id="com.instagram.android:id/button" />
|
||||
<node package="com.unknown.keyboard" resource-id="com.unknown.keyboard:id/key" />
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
# We shouldn't call LLM for foreign app
|
||||
with patch('GramAddict.core.llm_provider.query_telepathic_llm') as mock_llm:
|
||||
# Let's mock ScreenMemoryDB to return NORMAL
|
||||
with patch('GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type', return_value="NORMAL"):
|
||||
res = sae.perceive(xml)
|
||||
assert res == SituationType.NORMAL
|
||||
# The LLM for foreign app should NOT have been called.
|
||||
mock_llm.assert_not_called()
|
||||
@@ -166,10 +166,10 @@ class TestVerifySuccessExploreGrid:
|
||||
just didn't register; it doesn't mean the mapping is wrong.
|
||||
"""
|
||||
|
||||
def test_verify_success_returns_false_when_still_on_grid(self):
|
||||
def test_verify_success_returns_none_when_still_on_grid(self):
|
||||
"""
|
||||
If we tapped a grid item but the screen still shows the explore grid
|
||||
(no feed markers), verify_success must return False.
|
||||
(no feed markers), verify_success must return None (Inconclusive).
|
||||
"""
|
||||
engine = TelepathicEngine()
|
||||
|
||||
@@ -192,7 +192,7 @@ class TestVerifySuccessExploreGrid:
|
||||
"""
|
||||
|
||||
result = engine.verify_success("first image in explore grid", still_on_grid_xml)
|
||||
assert result is False, "verify_success should fail when still on explore grid"
|
||||
assert result is None, "verify_success should be inconclusive (None) when still on explore grid"
|
||||
|
||||
def test_verify_success_returns_true_when_post_opened(self):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user