feat(story_view): delegate next story segment resolution to TelepathicEngine/VLM instead of hardcoded coordinates

This commit is contained in:
2026-05-05 12:32:34 +02:00
parent 8729995338
commit 5ca5777c4b

View File

@@ -96,13 +96,44 @@ class StoryViewPlugin(BehaviorPlugin):
for i in range(count):
sleep(random.uniform(2.0, 5.0) * ctx.sleep_mod)
if i < count - 1:
# Click top-right to avoid 'reply' input fields and most stickers
humanized_click(ctx.device, int(w * 0.85), int(h * 0.25), sleep_mod=ctx.sleep_mod)
# Atomic state validation after click
# Atomic state validation before click
xml_dump = ctx.device.dump_hierarchy()
if not xml_dump:
continue
packages = set(re.findall(r'package="([^"]+)"', xml_dump))
# Query VLM to find the interactive area for the next segment
intent = "tap right side of screen to view next story segment"
node = ctx.telepathic.find_best_node(xml_dump, intent, device=ctx.device, track=False)
if node:
logger.debug(
f"📸 [StoryView] VLM selected node '{node.resource_id or node.content_desc or 'unknown'}' for next segment."
)
# If VLM selects a large container (e.g. the entire screen or story viewer),
# we must tap its right side, not its exact center, to avoid pausing the story.
if getattr(node, "area", 0) > (w * h * 0.4):
target_x = node.x1 + int((node.x2 - node.x1) * 0.85)
target_y = node.y1 + int((node.y2 - node.y1) * 0.25)
logger.debug(
f"📸 [StoryView] Adjusting click to top-right quadrant of large container: ({target_x}, {target_y})"
)
else:
target_x = node.center_x
target_y = node.center_y
humanized_click(ctx.device, target_x, target_y, sleep_mod=ctx.sleep_mod)
else:
logger.warning(
"📸 [StoryView] VLM could not resolve next story segment. Falling back to geometric safety quadrant."
)
# Click top-right to avoid 'reply' input fields and most stickers
humanized_click(ctx.device, int(w * 0.85), int(h * 0.25), sleep_mod=ctx.sleep_mod)
# Verify we didn't leave Instagram
xml_dump_after = ctx.device.dump_hierarchy()
if not xml_dump_after:
continue
packages = set(re.findall(r'package="([^"]+)"', xml_dump_after))
app_id = getattr(ctx.device, "app_id", "com.instagram.android")
if packages and app_id not in packages:
logger.error(