fix(perception): structural fast-path and VLM bounds extraction for Reels

This commit is contained in:
2026-05-02 23:17:12 +02:00
parent 91effbc843
commit e535c10b65
2 changed files with 159 additions and 12 deletions

View File

@@ -0,0 +1,105 @@
"""
Tests for PostDataExtractionPlugin
==================================
Ensures that data extraction robustly extracts the author username
from different UI layouts (Home Feed, Reels Feed, etc.) without failing.
"""
from GramAddict.core.behaviors.post_data_extraction import PostDataExtractionPlugin
from GramAddict.core.behaviors import BehaviorContext
from tests.e2e.conftest import load_fixture_xml
import pytest
class DummyDevice:
def dump_hierarchy(self):
return ""
def test_extract_username_from_home_feed():
plugin = PostDataExtractionPlugin()
xml = load_fixture_xml("home_feed_real.xml")
ctx = BehaviorContext(
device=DummyDevice(),
configs=None,
session_state=None,
cognitive_stack=None,
context_xml=xml,
post_data={},
username="",
)
result = plugin.execute(ctx)
assert result.executed is True
assert ctx.username != ""
assert ctx.username is not None
assert ctx.post_data.get("username_missing") is not True
def test_extract_username_from_reels_feed():
plugin = PostDataExtractionPlugin()
xml = load_fixture_xml("reels_feed_real.xml")
ctx = BehaviorContext(
device=DummyDevice(),
configs=None,
session_state=None,
cognitive_stack=None,
context_xml=xml,
post_data={},
username="",
)
result = plugin.execute(ctx)
assert result.executed is True, "PostDataExtraction failed to execute on Reels Feed"
assert ctx.username != "", "Username extracted from Reels Feed is empty!"
assert ctx.username is not None
assert ctx.post_data.get("username_missing") is not True
def test_extract_username_with_vlm_fallback(monkeypatch):
"""
Tests that if the structural fast path fails, the VLM fallback correctly
descends into ViewGroups if the VLM selects a container instead of the text node.
"""
plugin = PostDataExtractionPlugin()
xml = load_fixture_xml("reels_feed_real.xml")
# Mock TelepathicEngine to return the container node (like in the logs)
class FakeTelepath:
def find_best_node(self, xml_str, prompt, **kwargs):
if "author username" in prompt:
# Return the clips_author_info_component (a ViewGroup with no text)
return {
"original_attribs": {
"resource-id": "com.instagram.android:id/clips_author_info_component",
"text": "",
"content_desc": "",
"bounds": "[42,1957][591,2098]"
}
}
if "media content" in prompt:
return {"original_attribs": {"content_desc": "Test media"}}
return None
from GramAddict.core.telepathic_engine import TelepathicEngine
monkeypatch.setattr(TelepathicEngine, "get_instance", lambda *args, **kwargs: FakeTelepath())
# We must also mock the structural fast path so it falls through to VLM
# by temporarily removing 'clips_author_username' from the fast path list during this test.
# We will just mutate the xml to not have the structural IDs
xml_no_fastpath = xml.replace("row_feed_photo_profile_name", "hidden_id")
xml_no_fastpath = xml_no_fastpath.replace("clips_author_username", "hidden_id")
ctx = BehaviorContext(
device=DummyDevice(),
configs=None,
session_state=None,
cognitive_stack=None,
context_xml=xml_no_fastpath,
post_data={},
username="",
)
result = plugin.execute(ctx)
assert result.executed is True
# The VLM selected the container. The engine should have parsed the children
# and found "aditnugrahh" inside it.
assert ctx.username == "aditnugrahh", f"Expected 'aditnugrahh' but got {ctx.username}"