From 832eb8d45b6c1566988ee3b3969d99a418e97990 Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Mon, 4 May 2026 10:39:34 +0200 Subject: [PATCH] feat(ai): integrate Meta AI chip selection for automated commenting - Updated QNavGraph.do to accept **kwargs and forward them to GOAP. - Added explicit 'type and post comment' intent handler in GoalExecutor. - Implemented smart VLM-driven Meta AI tone selection. If 'Supportive', 'Funny', or other Meta AI chips are visible on the comment keyboard, the bot uses the VLM to pick the best tone chip and lets Meta AI write the comment, falling back to manual typing if no chips are found. --- GramAddict/core/goap.py | 75 +++++++++++++++++++++++++++++++++- GramAddict/core/q_nav_graph.py | 6 +-- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index 48676df..b24e9e8 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -329,7 +329,7 @@ class GoalExecutor: return False - def _execute_action(self, action: str, goal: str = None) -> bool: + def _execute_action(self, action: str, goal: str = None, **kwargs) -> bool: """Execute a single natural-language action using the TelepathicEngine.""" if action == "press back": @@ -349,6 +349,9 @@ class GoalExecutor: random_sleep(2.0, 3.5) return True + if action == "type and post comment": + return self._execute_type_and_post_comment(kwargs.get("text", "")) + # Use TelepathicEngine for any semantic click from GramAddict.core.telepathic_engine import TelepathicEngine @@ -555,6 +558,76 @@ class GoalExecutor: achieved = self.planner.plan_next_step(goal, screen) is None return achieved + def _execute_type_and_post_comment(self, fallback_text: str) -> bool: + """Handles the specific typing interaction, prioritizing Meta AI chips if available.""" + from GramAddict.core.telepathic_engine import TelepathicEngine + import xml.etree.ElementTree as ET + + logger.info("💬 [GOAP] Executing 'type and post comment'...") + engine = TelepathicEngine.get_instance() + + # 1. Tap the comment input field to open the keyboard + xml_dump = self.device.dump_hierarchy() + if "com.google.android.inputmethod.latin" not in xml_dump: + logger.info("⌨️ [GOAP] Tapping comment composer to open keyboard...") + best_node = engine.find_best_node(xml_dump, "tap comment input field", min_confidence=0.7, device=self.device) + if best_node: + self.device.click(obj=best_node) + random_sleep(1.5, 2.5) + xml_dump = self.device.dump_hierarchy() + else: + logger.warning("⚠️ [GOAP] Could not find comment input field.") + return False + + # 2. Check for Meta AI chips (Supportive, Funny, etc.) + meta_ai_chips = [] + try: + root = ET.fromstring(xml_dump.encode("utf-8")) + for node in root.iter("node"): + node_text = node.attrib.get("text", "") + if node_text in ["Supportive", "Rewrite", "Absurd", "Casual", "Funny", "Heartfelt", "Professional"]: + meta_ai_chips.append(node_text) + except Exception as e: + logger.debug(f"XML parse error for Meta AI chips: {e}") + + if meta_ai_chips: + logger.info(f"✨ [Meta AI] Detected Meta AI chips: {meta_ai_chips}") + logger.info("🧠 [Meta AI] Asking VLM to select the best tone...") + + # Use Telepathic Engine to pick the best chip via VLM + chip_node = engine.find_best_node( + xml_dump, + "tap the best Meta AI tone chip to generate a comment (e.g. Supportive, Funny, Casual)", + min_confidence=0.6, + device=self.device + ) + + if chip_node: + logger.info(f"✨ [Meta AI] VLM selected chip: '{chip_node.get('text', 'Unknown')}'") + self.device.click(obj=chip_node) + random_sleep(3.0, 4.5) # Wait for Meta AI to generate the text + else: + logger.warning("⚠️ [Meta AI] VLM failed to pick a chip, falling back to manual typing.") + self.device.type_text(fallback_text) + random_sleep(1.0, 2.0) + else: + # 3. Fallback: Type the text provided by our own VLM writer + logger.info(f"⌨️ [GOAP] Typing comment manually: {fallback_text}") + self.device.type_text(fallback_text) + random_sleep(1.0, 2.0) + + # 4. Click the Post button + xml_dump = self.device.dump_hierarchy() + post_btn = engine.find_best_node(xml_dump, "tap post comment button", min_confidence=0.7, device=self.device) + if post_btn: + self.device.click(obj=post_btn) + random_sleep(1.5, 2.5) + logger.info("✅ [GOAP] Comment posted successfully.") + return True + else: + logger.warning("⚠️ [GOAP] Could not find post button after typing.") + return False + # ── Convenience methods (backward compatibility with navigate_to) ── def navigate_to_screen(self, target: str) -> bool: diff --git a/GramAddict/core/q_nav_graph.py b/GramAddict/core/q_nav_graph.py index 3fa1917..7685231 100644 --- a/GramAddict/core/q_nav_graph.py +++ b/GramAddict/core/q_nav_graph.py @@ -116,7 +116,7 @@ class QNavGraph: return success - def do(self, goal: str) -> bool: + def do(self, goal: str, **kwargs) -> bool: """ GOAP-powered action execution. Replaces _execute_transition() for post interactions. @@ -124,10 +124,10 @@ class QNavGraph: Usage: nav_graph.do("like this post") # instead of _execute_transition("tap_like_button") nav_graph.do("follow this user") # instead of _execute_transition("tap_follow_button") - nav_graph.do("tap first grid item") # instead of _execute_transition("tap_explore_grid_item") + nav_graph.do("type and post comment", text="Nice!") """ - return self.goap._execute_action(goal) + return self.goap._execute_action(goal, **kwargs) def _find_path(self, start: str, end: str): """Delegates to ScreenTopology for BFS pathfinding (SSOT)."""