feat(story_view): delegate next story segment resolution to TelepathicEngine/VLM instead of hardcoded coordinates
This commit is contained in:
@@ -96,13 +96,44 @@ class StoryViewPlugin(BehaviorPlugin):
|
||||
for i in range(count):
|
||||
sleep(random.uniform(2.0, 5.0) * ctx.sleep_mod)
|
||||
if i < count - 1:
|
||||
# Click top-right to avoid 'reply' input fields and most stickers
|
||||
humanized_click(ctx.device, int(w * 0.85), int(h * 0.25), sleep_mod=ctx.sleep_mod)
|
||||
# Atomic state validation after click
|
||||
# Atomic state validation before click
|
||||
xml_dump = ctx.device.dump_hierarchy()
|
||||
if not xml_dump:
|
||||
continue
|
||||
packages = set(re.findall(r'package="([^"]+)"', xml_dump))
|
||||
|
||||
# Query VLM to find the interactive area for the next segment
|
||||
intent = "tap right side of screen to view next story segment"
|
||||
node = ctx.telepathic.find_best_node(xml_dump, intent, device=ctx.device, track=False)
|
||||
|
||||
if node:
|
||||
logger.debug(
|
||||
f"📸 [StoryView] VLM selected node '{node.resource_id or node.content_desc or 'unknown'}' for next segment."
|
||||
)
|
||||
# If VLM selects a large container (e.g. the entire screen or story viewer),
|
||||
# we must tap its right side, not its exact center, to avoid pausing the story.
|
||||
if getattr(node, "area", 0) > (w * h * 0.4):
|
||||
target_x = node.x1 + int((node.x2 - node.x1) * 0.85)
|
||||
target_y = node.y1 + int((node.y2 - node.y1) * 0.25)
|
||||
logger.debug(
|
||||
f"📸 [StoryView] Adjusting click to top-right quadrant of large container: ({target_x}, {target_y})"
|
||||
)
|
||||
else:
|
||||
target_x = node.center_x
|
||||
target_y = node.center_y
|
||||
|
||||
humanized_click(ctx.device, target_x, target_y, sleep_mod=ctx.sleep_mod)
|
||||
else:
|
||||
logger.warning(
|
||||
"📸 [StoryView] VLM could not resolve next story segment. Falling back to geometric safety quadrant."
|
||||
)
|
||||
# Click top-right to avoid 'reply' input fields and most stickers
|
||||
humanized_click(ctx.device, int(w * 0.85), int(h * 0.25), sleep_mod=ctx.sleep_mod)
|
||||
|
||||
# Verify we didn't leave Instagram
|
||||
xml_dump_after = ctx.device.dump_hierarchy()
|
||||
if not xml_dump_after:
|
||||
continue
|
||||
packages = set(re.findall(r'package="([^"]+)"', xml_dump_after))
|
||||
app_id = getattr(ctx.device, "app_id", "com.instagram.android")
|
||||
if packages and app_id not in packages:
|
||||
logger.error(
|
||||
|
||||
Reference in New Issue
Block a user