mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat(perception): VLM-driven Meta AI comment generation integration

Browse Source
This commit is contained in:
Marc Mintel
2026-05-04 11:45:41 +02:00
parent a67072eec4
commit 49f82d467f
2 changed files with 214 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
GramAddict/core/goap.py
View File

@@ -329,7 +329,7 @@ class GoalExecutor:
return False
def _execute_action(self, action: str, goal: str = None) -> bool:
def _execute_action(self, action: str, goal: str = None, **kwargs) -> bool:
"""Execute a single natural-language action using the TelepathicEngine."""
if action == "press back":
@@ -349,6 +349,9 @@ class GoalExecutor:
random_sleep(2.0, 3.5)
return True
if action == "type and post comment":
return self._execute_type_and_post_comment(kwargs.get("text", ""))
# Use TelepathicEngine for any semantic click
from GramAddict.core.telepathic_engine import TelepathicEngine
@@ -555,6 +558,83 @@ class GoalExecutor:
achieved = self.planner.plan_next_step(goal, screen) is None
return achieved
def _execute_type_and_post_comment(self, fallback_text: str) -> bool:
"""Handles the specific typing interaction, prioritizing Meta AI chips if available."""
import xml.etree.ElementTree as ET
from GramAddict.core.telepathic_engine import TelepathicEngine
logger.info("💬 [GOAP] Executing 'type and post comment'...")
engine = TelepathicEngine.get_instance()
# 1. Tap the comment input field to open the keyboard
xml_dump = self.device.dump_hierarchy()
if "com.google.android.inputmethod.latin" not in xml_dump:
logger.info("⌨️ [GOAP] Tapping comment composer to open keyboard...")
best_node = engine.find_best_node(
xml_dump, "tap comment input field", min_confidence=0.7, device=self.device
)
if best_node:
self.device.click(obj=best_node)
random_sleep(1.5, 2.5)
xml_dump = self.device.dump_hierarchy()
else:
logger.warning("⚠️ [GOAP] Could not find comment input field.")
return False
# 2. Check for Meta AI chips (Supportive, Funny, etc.)
meta_ai_chips = []
try:
root = ET.fromstring(xml_dump.encode("utf-8"))
for node in root.iter("node"):
node_text = node.attrib.get("text", "")
if node_text in ["Supportive", "Rewrite", "Absurd", "Casual", "Funny", "Heartfelt", "Professional"]:
meta_ai_chips.append(node_text)
except Exception as e:
logger.debug(f"XML parse error for Meta AI chips: {e}")
if meta_ai_chips:
logger.info(f"✨ [Meta AI] Detected Meta AI chips: {meta_ai_chips}")
logger.info("🧠 [Meta AI] Asking VLM to select the best tone...")
# Use Telepathic Engine to pick the best chip via VLM
chip_node = engine.find_best_node(
xml_dump,
"tap the best Meta AI tone chip to generate a comment (e.g. Supportive, Funny, Casual)",
min_confidence=0.6,
device=self.device,
)
if chip_node:
logger.info(f"✨ [Meta AI] VLM selected chip: '{chip_node.get('text', 'Unknown')}'")
self.device.click(obj=chip_node)
random_sleep(3.0, 4.5) # Wait for Meta AI to generate the text
else:
logger.warning("⚠️ [Meta AI] VLM failed to pick a chip, falling back to manual typing.")
from GramAddict.core.stealth_typing import ghost_type
ghost_type(self.device, fallback_text)
random_sleep(1.0, 2.0)
else:
# 3. Fallback: Type the text provided by our own VLM writer
logger.info(f"⌨️ [GOAP] Typing comment manually: {fallback_text}")
from GramAddict.core.stealth_typing import ghost_type
ghost_type(self.device, fallback_text)
random_sleep(1.0, 2.0)
# 4. Click the Post button
xml_dump = self.device.dump_hierarchy()
post_btn = engine.find_best_node(xml_dump, "tap post comment button", min_confidence=0.7, device=self.device)
if post_btn:
self.device.click(obj=post_btn)
random_sleep(1.5, 2.5)
logger.info("✅ [GOAP] Comment posted successfully.")
return True
else:
logger.warning("⚠️ [GOAP] Could not find post button after typing.")
return False
# ── Convenience methods (backward compatibility with navigate_to) ──
def navigate_to_screen(self, target: str) -> bool:

133
tests/e2e/test_goap_meta_ai_comment.py Normal file
View File

@@ -0,0 +1,133 @@
import pytest
@pytest.mark.live_llm
def test_goap_meta_ai_comment_selection(make_real_device_with_image):
"""
TDD Test: Verifies that when the comment keyboard is open and Meta AI chips
are present on the screen, the GOAP engine detects them and uses the VLM
to select the best tone chip, entirely bypassing manual fallback typing.
"""
from GramAddict.core.goap import GoalExecutor
# Define an XML hierarchy that simulates the comment keyboard open with Meta AI chips.
xml_chip_first = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node package="com.instagram.android" class="android.widget.FrameLayout" bounds="[0,0][1080,2400]">
<!-- Meta AI Chips FIRST so they become box 0 for the VLM mock -->
<node class="android.widget.TextView" text="Funny" bounds="[450,1150][700,1250]" clickable="true" />
<node class="android.widget.TextView" text="Supportive" bounds="[100,1150][400,1250]" clickable="true" />
<node class="android.widget.TextView" text="Rewrite" bounds="[750,1150][950,1250]" clickable="true" />
<!-- The comment composer input field -->
<node resource-id="com.instagram.android:id/layout_comment_thread_edittext" text="Add a comment..." bounds="[100,1000][900,1100]" clickable="true" />
<!-- The Post button -->
<node resource-id="com.instagram.android:id/layout_comment_thread_button_post" text="Post" bounds="[900,1000][1000,1100]" clickable="true" />
</node>
<!-- Keyboard is open -->
<node package="com.google.android.inputmethod.latin" class="android.widget.FrameLayout" bounds="[0,1500][1080,2400]">
<node resource-id="com.google.android.inputmethod.latin:id/B00" content-desc="Q" bounds="[0,1800][100,1900]" clickable="true" />
</node>
</hierarchy>
"""
xml_post_first = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node package="com.instagram.android" class="android.widget.FrameLayout" bounds="[0,0][1080,2400]">
<!-- Post button FIRST so it becomes box 0 for the VLM mock -->
<node resource-id="com.instagram.android:id/layout_comment_thread_button_post" text="Post" bounds="[900,1000][1000,1100]" clickable="true" />
<!-- The comment composer input field -->
<node resource-id="com.instagram.android:id/layout_comment_thread_edittext" text="Add a comment..." bounds="[100,1000][900,1100]" clickable="true" />
<!-- Meta AI Chips -->
<node class="android.widget.TextView" text="Funny" bounds="[450,1150][700,1250]" clickable="true" />
<node class="android.widget.TextView" text="Supportive" bounds="[100,1150][400,1250]" clickable="true" />
</node>
</hierarchy>
"""
# We provide this XML twice: once for the initial check (picks chip), once for the 'Post' button click.
device = make_real_device_with_image(
"tests/fixtures/home_feed_with_ad.jpg", [xml_chip_first, xml_post_first, xml_post_first]
)
# To track if ghost_type was incorrectly called
type_text_calls = []
def mock_ghost_type(dev, text, speed="normal"):
type_text_calls.append(text)
# Monkeypatch the module where ghost_type is imported, or just the whole module.
# Actually, goap.py imports ghost_type dynamically:
# `from GramAddict.core.stealth_typing import ghost_type`
# So we monkeypatch it in stealth_typing
import GramAddict.core.stealth_typing
original_ghost_type = GramAddict.core.stealth_typing.ghost_type
GramAddict.core.stealth_typing.ghost_type = mock_ghost_type
import GramAddict.core.telepathic_engine
original_find_best_node = GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node
def mock_find_best_node(self, xml_dump, instruction, *args, **kwargs):
# We parse the XML to return a specific node based on the instruction
import re
import xml.etree.ElementTree as ET
root = ET.fromstring(xml_dump.encode("utf-8"))
def _enrich_node(node):
attribs = dict(node.attrib)
bounds_str = attribs.get("bounds", "")
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
if match:
left, top, right, bottom = map(int, match.groups())
attribs["x"] = (left + right) // 2
attribs["y"] = (top + bottom) // 2
return attribs
if "tap the best Meta AI tone chip" in instruction:
for node in root.iter("node"):
if node.attrib.get("text") == "Funny":
return _enrich_node(node)
return None
elif "tap post comment button" in instruction:
for node in root.iter("node"):
if node.attrib.get("text") == "Post":
return _enrich_node(node)
return None
return None
GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node = mock_find_best_node
try:
goap = GoalExecutor.get_instance(device, bot_username="testuser")
# Execute the new GOAP action
result = goap._execute_action("type and post comment", text="This is a fallback text")
finally:
# Cleanup mocks
GramAddict.core.stealth_typing.ghost_type = original_ghost_type
GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node = original_find_best_node
# Assertions
assert result is True, "GOAP action 'type and post comment' failed."
# We expect 2 clicks:
# 1. Tapping one of the Meta AI chips (y between 1150 and 1250)
# 2. Tapping the "Post" button (y between 1000 and 1100)
# (Since keyboard is already open, it shouldn't tap the input field)
assert len(device.clicks) >= 2, f"Expected at least 2 clicks (Meta AI chip + Post), got {len(device.clicks)}"
print(f"DEBUG: device.clicks = {device.clicks}")
# Ensure a Meta AI chip was clicked (y between 1150 and 1250)
chip_clicked = any(1150 <= y <= 1250 for (x, y) in device.clicks)
assert chip_clicked, f"VLM failed to select and click a Meta AI chip! Clicks were: {device.clicks}"
# Ensure manual typing was bypassed!
assert len(type_text_calls) == 0, f"Expected 0 ghost_type calls, but got: {type_text_calls}"
# Cleanup
GramAddict.core.stealth_typing.ghost_type = original_ghost_type