feat(perception): VLM-driven Meta AI comment generation integration
This commit is contained in:
133
tests/e2e/test_goap_meta_ai_comment.py
Normal file
133
tests/e2e/test_goap_meta_ai_comment.py
Normal file
@@ -0,0 +1,133 @@
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_goap_meta_ai_comment_selection(make_real_device_with_image):
|
||||
"""
|
||||
TDD Test: Verifies that when the comment keyboard is open and Meta AI chips
|
||||
are present on the screen, the GOAP engine detects them and uses the VLM
|
||||
to select the best tone chip, entirely bypassing manual fallback typing.
|
||||
"""
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
# Define an XML hierarchy that simulates the comment keyboard open with Meta AI chips.
|
||||
xml_chip_first = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node package="com.instagram.android" class="android.widget.FrameLayout" bounds="[0,0][1080,2400]">
|
||||
<!-- Meta AI Chips FIRST so they become box 0 for the VLM mock -->
|
||||
<node class="android.widget.TextView" text="Funny" bounds="[450,1150][700,1250]" clickable="true" />
|
||||
<node class="android.widget.TextView" text="Supportive" bounds="[100,1150][400,1250]" clickable="true" />
|
||||
<node class="android.widget.TextView" text="Rewrite" bounds="[750,1150][950,1250]" clickable="true" />
|
||||
|
||||
<!-- The comment composer input field -->
|
||||
<node resource-id="com.instagram.android:id/layout_comment_thread_edittext" text="Add a comment..." bounds="[100,1000][900,1100]" clickable="true" />
|
||||
<!-- The Post button -->
|
||||
<node resource-id="com.instagram.android:id/layout_comment_thread_button_post" text="Post" bounds="[900,1000][1000,1100]" clickable="true" />
|
||||
</node>
|
||||
|
||||
<!-- Keyboard is open -->
|
||||
<node package="com.google.android.inputmethod.latin" class="android.widget.FrameLayout" bounds="[0,1500][1080,2400]">
|
||||
<node resource-id="com.google.android.inputmethod.latin:id/B00" content-desc="Q" bounds="[0,1800][100,1900]" clickable="true" />
|
||||
</node>
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
xml_post_first = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node package="com.instagram.android" class="android.widget.FrameLayout" bounds="[0,0][1080,2400]">
|
||||
<!-- Post button FIRST so it becomes box 0 for the VLM mock -->
|
||||
<node resource-id="com.instagram.android:id/layout_comment_thread_button_post" text="Post" bounds="[900,1000][1000,1100]" clickable="true" />
|
||||
<!-- The comment composer input field -->
|
||||
<node resource-id="com.instagram.android:id/layout_comment_thread_edittext" text="Add a comment..." bounds="[100,1000][900,1100]" clickable="true" />
|
||||
|
||||
<!-- Meta AI Chips -->
|
||||
<node class="android.widget.TextView" text="Funny" bounds="[450,1150][700,1250]" clickable="true" />
|
||||
<node class="android.widget.TextView" text="Supportive" bounds="[100,1150][400,1250]" clickable="true" />
|
||||
</node>
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
# We provide this XML twice: once for the initial check (picks chip), once for the 'Post' button click.
|
||||
device = make_real_device_with_image(
|
||||
"tests/fixtures/home_feed_with_ad.jpg", [xml_chip_first, xml_post_first, xml_post_first]
|
||||
)
|
||||
|
||||
# To track if ghost_type was incorrectly called
|
||||
type_text_calls = []
|
||||
|
||||
def mock_ghost_type(dev, text, speed="normal"):
|
||||
type_text_calls.append(text)
|
||||
|
||||
# Monkeypatch the module where ghost_type is imported, or just the whole module.
|
||||
# Actually, goap.py imports ghost_type dynamically:
|
||||
# `from GramAddict.core.stealth_typing import ghost_type`
|
||||
# So we monkeypatch it in stealth_typing
|
||||
import GramAddict.core.stealth_typing
|
||||
|
||||
original_ghost_type = GramAddict.core.stealth_typing.ghost_type
|
||||
GramAddict.core.stealth_typing.ghost_type = mock_ghost_type
|
||||
|
||||
import GramAddict.core.telepathic_engine
|
||||
|
||||
original_find_best_node = GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node
|
||||
|
||||
def mock_find_best_node(self, xml_dump, instruction, *args, **kwargs):
|
||||
# We parse the XML to return a specific node based on the instruction
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
root = ET.fromstring(xml_dump.encode("utf-8"))
|
||||
|
||||
def _enrich_node(node):
|
||||
attribs = dict(node.attrib)
|
||||
bounds_str = attribs.get("bounds", "")
|
||||
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
|
||||
if match:
|
||||
left, top, right, bottom = map(int, match.groups())
|
||||
attribs["x"] = (left + right) // 2
|
||||
attribs["y"] = (top + bottom) // 2
|
||||
return attribs
|
||||
|
||||
if "tap the best Meta AI tone chip" in instruction:
|
||||
for node in root.iter("node"):
|
||||
if node.attrib.get("text") == "Funny":
|
||||
return _enrich_node(node)
|
||||
return None
|
||||
elif "tap post comment button" in instruction:
|
||||
for node in root.iter("node"):
|
||||
if node.attrib.get("text") == "Post":
|
||||
return _enrich_node(node)
|
||||
return None
|
||||
return None
|
||||
|
||||
GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node = mock_find_best_node
|
||||
|
||||
try:
|
||||
goap = GoalExecutor.get_instance(device, bot_username="testuser")
|
||||
|
||||
# Execute the new GOAP action
|
||||
result = goap._execute_action("type and post comment", text="This is a fallback text")
|
||||
finally:
|
||||
# Cleanup mocks
|
||||
GramAddict.core.stealth_typing.ghost_type = original_ghost_type
|
||||
GramAddict.core.telepathic_engine.TelepathicEngine.find_best_node = original_find_best_node
|
||||
|
||||
# Assertions
|
||||
assert result is True, "GOAP action 'type and post comment' failed."
|
||||
|
||||
# We expect 2 clicks:
|
||||
# 1. Tapping one of the Meta AI chips (y between 1150 and 1250)
|
||||
# 2. Tapping the "Post" button (y between 1000 and 1100)
|
||||
# (Since keyboard is already open, it shouldn't tap the input field)
|
||||
assert len(device.clicks) >= 2, f"Expected at least 2 clicks (Meta AI chip + Post), got {len(device.clicks)}"
|
||||
|
||||
print(f"DEBUG: device.clicks = {device.clicks}")
|
||||
# Ensure a Meta AI chip was clicked (y between 1150 and 1250)
|
||||
chip_clicked = any(1150 <= y <= 1250 for (x, y) in device.clicks)
|
||||
assert chip_clicked, f"VLM failed to select and click a Meta AI chip! Clicks were: {device.clicks}"
|
||||
|
||||
# Ensure manual typing was bypassed!
|
||||
assert len(type_text_calls) == 0, f"Expected 0 ghost_type calls, but got: {type_text_calls}"
|
||||
|
||||
# Cleanup
|
||||
GramAddict.core.stealth_typing.ghost_type = original_ghost_type
|
||||
Reference in New Issue
Block a user