fix(perception): active keyboard dismiss guard and POST_DETAIL structural share button
- Added active Keyboard Dismiss logic to TelepathicEngine.find_best_node(): If the keyboard is open and the intent is NOT typing-related (e.g., 'tap post username'), the bot automatically issues a back press to close the keyboard and re-fetches the UI state. - Broadened structural fast-path for 'tap send post button' in IntentResolver to also match 'direct_share_button' or 'send'/'share' desc, preventing VLM fallback on POST_DETAIL screens (Bug #3).
This commit is contained in:
@@ -310,7 +310,8 @@ class IntentResolver:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row_feed_button_share" in rid or "send post" in desc:
|
||||
# Feed uses row_feed_button_share, POST_DETAIL may use direct_share_button or just desc "Send"
|
||||
if "row_feed_button_share" in rid or "direct_share_button" in rid or "send" in desc or "share" in desc:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
return node
|
||||
|
||||
|
||||
@@ -85,6 +85,23 @@ class TelepathicEngine:
|
||||
filtered_candidates.append(c)
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Active Keyboard Dismiss Guard ---
|
||||
# If keyboard is open but intent is NOT typing related -> dismiss it!
|
||||
intent_lower = intent_description.lower()
|
||||
if device and self._resolver.has_keyboard_open(candidates):
|
||||
typing_keywords = ["type", "message", "comment", "search", "write"]
|
||||
if not any(k in intent_lower for k in typing_keywords):
|
||||
logger.warning("⌨️ [TelepathicEngine] Keyboard detected during non-typing intent! Auto-dismissing.")
|
||||
import time
|
||||
device.back()
|
||||
time.sleep(1.0)
|
||||
# Re-fetch UI state
|
||||
xml_string = device.get_hierarchy()
|
||||
if xml_string:
|
||||
root = self._parser.parse(xml_string)
|
||||
if root:
|
||||
candidates = self._parser.get_clickable_nodes(root)
|
||||
|
||||
# 3. Resolve intent against candidates
|
||||
best_node = self._resolver.resolve(intent_description, candidates, device=device)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user