feat(ai): integrate Meta AI chip selection for automated commenting
- Updated QNavGraph.do to accept **kwargs and forward them to GOAP. - Added explicit 'type and post comment' intent handler in GoalExecutor. - Implemented smart VLM-driven Meta AI tone selection. If 'Supportive', 'Funny', or other Meta AI chips are visible on the comment keyboard, the bot uses the VLM to pick the best tone chip and lets Meta AI write the comment, falling back to manual typing if no chips are found.
This commit is contained in:
@@ -329,7 +329,7 @@ class GoalExecutor:
|
||||
|
||||
return False
|
||||
|
||||
def _execute_action(self, action: str, goal: str = None) -> bool:
|
||||
def _execute_action(self, action: str, goal: str = None, **kwargs) -> bool:
|
||||
"""Execute a single natural-language action using the TelepathicEngine."""
|
||||
|
||||
if action == "press back":
|
||||
@@ -349,6 +349,9 @@ class GoalExecutor:
|
||||
random_sleep(2.0, 3.5)
|
||||
return True
|
||||
|
||||
if action == "type and post comment":
|
||||
return self._execute_type_and_post_comment(kwargs.get("text", ""))
|
||||
|
||||
# Use TelepathicEngine for any semantic click
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
@@ -555,6 +558,76 @@ class GoalExecutor:
|
||||
achieved = self.planner.plan_next_step(goal, screen) is None
|
||||
return achieved
|
||||
|
||||
def _execute_type_and_post_comment(self, fallback_text: str) -> bool:
|
||||
"""Handles the specific typing interaction, prioritizing Meta AI chips if available."""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
logger.info("💬 [GOAP] Executing 'type and post comment'...")
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
# 1. Tap the comment input field to open the keyboard
|
||||
xml_dump = self.device.dump_hierarchy()
|
||||
if "com.google.android.inputmethod.latin" not in xml_dump:
|
||||
logger.info("⌨️ [GOAP] Tapping comment composer to open keyboard...")
|
||||
best_node = engine.find_best_node(xml_dump, "tap comment input field", min_confidence=0.7, device=self.device)
|
||||
if best_node:
|
||||
self.device.click(obj=best_node)
|
||||
random_sleep(1.5, 2.5)
|
||||
xml_dump = self.device.dump_hierarchy()
|
||||
else:
|
||||
logger.warning("⚠️ [GOAP] Could not find comment input field.")
|
||||
return False
|
||||
|
||||
# 2. Check for Meta AI chips (Supportive, Funny, etc.)
|
||||
meta_ai_chips = []
|
||||
try:
|
||||
root = ET.fromstring(xml_dump.encode("utf-8"))
|
||||
for node in root.iter("node"):
|
||||
node_text = node.attrib.get("text", "")
|
||||
if node_text in ["Supportive", "Rewrite", "Absurd", "Casual", "Funny", "Heartfelt", "Professional"]:
|
||||
meta_ai_chips.append(node_text)
|
||||
except Exception as e:
|
||||
logger.debug(f"XML parse error for Meta AI chips: {e}")
|
||||
|
||||
if meta_ai_chips:
|
||||
logger.info(f"✨ [Meta AI] Detected Meta AI chips: {meta_ai_chips}")
|
||||
logger.info("🧠 [Meta AI] Asking VLM to select the best tone...")
|
||||
|
||||
# Use Telepathic Engine to pick the best chip via VLM
|
||||
chip_node = engine.find_best_node(
|
||||
xml_dump,
|
||||
"tap the best Meta AI tone chip to generate a comment (e.g. Supportive, Funny, Casual)",
|
||||
min_confidence=0.6,
|
||||
device=self.device
|
||||
)
|
||||
|
||||
if chip_node:
|
||||
logger.info(f"✨ [Meta AI] VLM selected chip: '{chip_node.get('text', 'Unknown')}'")
|
||||
self.device.click(obj=chip_node)
|
||||
random_sleep(3.0, 4.5) # Wait for Meta AI to generate the text
|
||||
else:
|
||||
logger.warning("⚠️ [Meta AI] VLM failed to pick a chip, falling back to manual typing.")
|
||||
self.device.type_text(fallback_text)
|
||||
random_sleep(1.0, 2.0)
|
||||
else:
|
||||
# 3. Fallback: Type the text provided by our own VLM writer
|
||||
logger.info(f"⌨️ [GOAP] Typing comment manually: {fallback_text}")
|
||||
self.device.type_text(fallback_text)
|
||||
random_sleep(1.0, 2.0)
|
||||
|
||||
# 4. Click the Post button
|
||||
xml_dump = self.device.dump_hierarchy()
|
||||
post_btn = engine.find_best_node(xml_dump, "tap post comment button", min_confidence=0.7, device=self.device)
|
||||
if post_btn:
|
||||
self.device.click(obj=post_btn)
|
||||
random_sleep(1.5, 2.5)
|
||||
logger.info("✅ [GOAP] Comment posted successfully.")
|
||||
return True
|
||||
else:
|
||||
logger.warning("⚠️ [GOAP] Could not find post button after typing.")
|
||||
return False
|
||||
|
||||
# ── Convenience methods (backward compatibility with navigate_to) ──
|
||||
|
||||
def navigate_to_screen(self, target: str) -> bool:
|
||||
|
||||
@@ -116,7 +116,7 @@ class QNavGraph:
|
||||
|
||||
return success
|
||||
|
||||
def do(self, goal: str) -> bool:
|
||||
def do(self, goal: str, **kwargs) -> bool:
|
||||
"""
|
||||
GOAP-powered action execution.
|
||||
Replaces _execute_transition() for post interactions.
|
||||
@@ -124,10 +124,10 @@ class QNavGraph:
|
||||
Usage:
|
||||
nav_graph.do("like this post") # instead of _execute_transition("tap_like_button")
|
||||
nav_graph.do("follow this user") # instead of _execute_transition("tap_follow_button")
|
||||
nav_graph.do("tap first grid item") # instead of _execute_transition("tap_explore_grid_item")
|
||||
nav_graph.do("type and post comment", text="Nice!")
|
||||
"""
|
||||
|
||||
return self.goap._execute_action(goal)
|
||||
return self.goap._execute_action(goal, **kwargs)
|
||||
|
||||
def _find_path(self, start: str, end: str):
|
||||
"""Delegates to ScreenTopology for BFS pathfinding (SSOT)."""
|
||||
|
||||
Reference in New Issue
Block a user