Compare commits
133 Commits
fix/e2e-te
...
f384fbb749
| Author | SHA1 | Date | |
|---|---|---|---|
| f384fbb749 | |||
| 565bdaa568 | |||
| c641204a6b | |||
| 4b645c6fb2 | |||
| c7c7ce29f8 | |||
| b36dde77d8 | |||
| 93b2140844 | |||
| 67c3d464e0 | |||
| d298f03891 | |||
| 604f2d7341 | |||
| cd8f35056c | |||
| 800fb1da98 | |||
| 6cd068f951 | |||
| f46b0b7bcb | |||
| 5fbbe3d273 | |||
| f85d0a8a76 | |||
| f0a54d4e20 | |||
| 0a73c35809 | |||
| e535c10b65 | |||
| 91effbc843 | |||
| 3da3849ca1 | |||
| 51ee7a6793 | |||
| 936da47f61 | |||
| d1e0995148 | |||
| 2f8eebb7e9 | |||
| 9c6f80de9d | |||
| cff7e976e0 | |||
| f6f15ebd9a | |||
| 1cc367697e | |||
| 738a59ac8d | |||
| cd6cecbe27 | |||
| 4af4ddb060 | |||
| f32ee46d8c | |||
| d2de5f91de | |||
| 9a13216064 | |||
| aa5184786e | |||
| 2e1edec56a | |||
| df18a48a84 | |||
| f1a8573be8 | |||
| da7201117c | |||
| fddf14fd67 | |||
| 392abff313 | |||
| 556bd181fa | |||
| 2c44331f03 | |||
| b83b55e02b | |||
| db226ed7c2 | |||
| 47f94b699c | |||
| 96fdbd7db7 | |||
| ca91ae4b33 | |||
| a560225dc9 | |||
| 849fb63426 | |||
| fc44633ebc | |||
| 0f5b71708d | |||
| 0ef2840f79 | |||
| 068a6a616a | |||
| effb1f5ae1 | |||
| 0e43996ccd | |||
| b6846ab0fe | |||
| 6db579f45b | |||
| 0ed12303ac | |||
| 6abb519e3b | |||
| 4e91db01c9 | |||
| e55abc5a8a | |||
| 03105437b8 | |||
| b9c29a5a2d | |||
| 71310b8e84 | |||
| 073a90c38c | |||
| 44fae37cc7 | |||
| 48071cc9b8 | |||
| 10a85a91f1 | |||
| 5bf0053884 | |||
| a846462d02 | |||
| 0dbafd0a82 | |||
| 83e5b94ddf | |||
| dd8285e1ce | |||
| ac5d5351a6 | |||
| ad012b4cd4 | |||
| 5fcf1f180b | |||
| 9a74d89477 | |||
| dc4b576bc1 | |||
| e94dfe8c5c | |||
| 7aa6bfccf6 | |||
| 5fef014cb4 | |||
| 0bdfd999d2 | |||
| 4ad559e107 | |||
| f220e09193 | |||
| de2a1c104f | |||
| 52c553827f | |||
| cd64794f55 | |||
| bd9148e6e9 | |||
| 1e1bba6b16 | |||
| 2b992cf2a8 | |||
| c051c3a4c3 | |||
| 3006020106 | |||
| 3b9465a3bc | |||
| 5d50228945 | |||
| 7277f27fae | |||
| ee3de811d3 | |||
| c93333928a | |||
| 12937cb2c1 | |||
| 097a5753f9 | |||
| 9ee6aab831 | |||
| da804b174a | |||
| 93175b7caf | |||
| e37d92cdfd | |||
| 1c38dabe79 | |||
| 7b8daa7670 | |||
| a7449a1db3 | |||
| 746eeb767d | |||
| 36a8683643 | |||
| 888136f733 | |||
| ae36b6e196 | |||
| e70ce0f52d | |||
| 22ca93c988 | |||
| 740f8f1f56 | |||
| f148efd2a0 | |||
| ac95dec9d8 | |||
| 0b68d4bc77 | |||
| 8c37290bc3 | |||
| b4bafb59be | |||
| 41450c4eaf | |||
| e9201e0e30 | |||
| ae046be3b1 | |||
| a2a4a75603 | |||
| 714c914432 | |||
| 294403d590 | |||
| 117e7a22e7 | |||
| 0fbd1b1678 | |||
| b5cca06ce2 | |||
| 3c4dd84a61 | |||
| 9ad49500f9 | |||
| 4de087ae45 | |||
| 42a11107fd |
18
.gitignore
vendored
18
.gitignore
vendored
@@ -10,6 +10,12 @@
|
||||
!test_config.yml
|
||||
*.json
|
||||
*.xml
|
||||
!tests/fixtures/*.xml
|
||||
!tests/fixtures/*.jpg
|
||||
!tests/fixtures/*.json
|
||||
!tests/e2e/fixtures/*.xml
|
||||
!tests/e2e/fixtures/*.jpg
|
||||
!tests/e2e/fixtures/*.json
|
||||
logs/
|
||||
*.pyc
|
||||
__pycache__/
|
||||
@@ -25,8 +31,14 @@ Pipfile.lock
|
||||
*.ini
|
||||
*.db
|
||||
|
||||
# Debug artifacts
|
||||
# Debug artifacts & garbage scripts (Rule 5: KRIEG DEM MÜLL)
|
||||
scratch*.py
|
||||
rewrite_*.py
|
||||
test_*.py
|
||||
!tests/**
|
||||
update_*.py
|
||||
profile_dump.*
|
||||
resp_dump.*
|
||||
test_compress.py
|
||||
test_fixtures.py
|
||||
output.txt
|
||||
@@ -37,3 +49,7 @@ traceback.log
|
||||
htmlcov/
|
||||
.coverage
|
||||
coverage.xml
|
||||
.hypothesis/
|
||||
|
||||
# Local diagnostic traces
|
||||
debug/
|
||||
|
||||
Binary file not shown.
@@ -29,6 +29,13 @@ Found in `sensors/honeypot_radome.py`.
|
||||
- **Ghost Engagement Guard**: Strips DOM nodes explicitly tagged with `visible-to-user="false"` to prevent triggering Accessibility Hooks.
|
||||
- **VLM Sanity Guard**: Woven into `telepathic_engine.py`, it sends semantic matches for destructive actions (Like/Follow) through a Vision Language Model step to prevent executing semantic "Bait and Switch" tricks.
|
||||
|
||||
### 🧠 Situational Awareness Engine (SAE)
|
||||
Found in `situational_awareness.py`. Handles autonomous obstacle detection, recovery, and learning without hardcoded rules.
|
||||
- **3-Layer Modal Fast-Path**: Eliminates LLM hallucination traps for Instagram-internal modals (surveys, rating prompts) via O(1) deterministic structural checks:
|
||||
1. **Resource-ID Guard**: Detects internal blocking overlays (e.g., `survey_overlay_container`, `nux_overlay`).
|
||||
2. **Dismiss-Button Heuristic**: Cross-validates typical negative actions ("Not Now", "Take Survey") with overlay structures to prevent false positives in post captions.
|
||||
3. **Zero-Deception Fallback**: If structural markers fail, falls back to `ScreenMemoryDB` and ultimately the LLM. Structured invariants always override the semantic cache.
|
||||
|
||||
### 🦾 Biometric Facade (Gaussian Clicks)
|
||||
Found in `device_facade.py`.
|
||||
- Human touches do not follow a flat mathematical uniform grid. The GramPilot simulates genuine **biometric dispersion** using `random.gauss(mu, sigma)`, strictly centering clicks inside a thumb-bias radius (bottom-left skew for right-handers). In tests, this hits a 68% standard deviation precision.
|
||||
@@ -37,3 +44,9 @@ Found in `device_facade.py`.
|
||||
Instead of hardcoding limits like `max_likes = 50`, the bot stops interacting based on **simulated boredom**.
|
||||
- The `ResonanceEngine` calculates the aesthetic score of content.
|
||||
- The `DopamineEngine` uses this score to modulate pace. High resonance = engagement. Low resonance over multiple posts = early session termination (simulating human fatigue).
|
||||
|
||||
## 4. The 100% Autonomy Directive (Zero Hardcoding)
|
||||
GramPilot is designed as a true agent, not a state-machine script. It operates on **absolute zero hardcoded UI states or edge cases**.
|
||||
- **No Manual Guards**: Features like `if "row_feed_button_like" not in xml:` or `if state == "ReelsFeed":` are strictly prohibited. The bot must understand the screen via its Vision-Language-Action (VLA) pipeline.
|
||||
- **No Hand-Holding**: If the LLM makes a mistake (e.g., clicking the wrong button in a DM), the solution is to improve the VLM prompt, the system architecture, or the Visual Critic. We never insert `if is_dm_thread:` hacks.
|
||||
- **Smart like a human**: The bot navigates by visually confirming targets, detecting obstacles when the UI organically stops responding, and inferring context precisely like a real user scrolling.
|
||||
|
||||
@@ -47,7 +47,7 @@ def verify_and_switch_account(device, nav_graph, target_username):
|
||||
telepath = TelepathicEngine.get_instance()
|
||||
|
||||
# We ask the semantic engine to find the profile tab, ensuring 100% ID-agnostic behavior
|
||||
profile_tab_node = telepath.find_best_node(xml_dump, "tap profile tab", min_threshold=0.3)
|
||||
profile_tab_node = telepath.find_best_node(xml_dump, "tap profile tab", min_threshold=0.3, device=device)
|
||||
if profile_tab_node:
|
||||
profile_tab = (profile_tab_node["x"], profile_tab_node["y"])
|
||||
except Exception as e:
|
||||
@@ -113,7 +113,7 @@ def verify_and_switch_account(device, nav_graph, target_username):
|
||||
dump_ui_state(
|
||||
device, "identity_guard", {"reason": "account_not_found_in_bottom_sheet", "target": target_username}
|
||||
)
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
# Escape the bottom sheet
|
||||
device.press("back")
|
||||
|
||||
@@ -240,6 +240,7 @@ from GramAddict.core.behaviors.profile_visit import ProfileVisitPlugin # noqa:
|
||||
from GramAddict.core.behaviors.rabbit_hole import RabbitHolePlugin # noqa: E402
|
||||
from GramAddict.core.behaviors.repost import RepostPlugin # noqa: E402
|
||||
from GramAddict.core.behaviors.resonance_evaluator import ResonanceEvaluatorPlugin # noqa: E402
|
||||
from GramAddict.core.behaviors.scrape_profile import ScrapeProfilePlugin # noqa: E402
|
||||
|
||||
# Note: We do not automatically instantiate all of them globally here to avoid circular
|
||||
# dependencies during initial load. The bot_flow.py engine should explicitly register them.
|
||||
@@ -265,3 +266,4 @@ def load_all_plugins():
|
||||
registry.register(RabbitHolePlugin())
|
||||
registry.register(RepostPlugin())
|
||||
registry.register(ResonanceEvaluatorPlugin())
|
||||
registry.register(ScrapeProfilePlugin())
|
||||
|
||||
@@ -31,7 +31,7 @@ class AnomalyHandlerPlugin(BehaviorPlugin):
|
||||
return getattr(self, "_enabled", True)
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
telepathic = ctx.cognitive_stack.get("telepathic") or TelepathicEngine.get_instance()
|
||||
xml = ctx.context_xml if ctx.context_xml else ctx.device.dump_hierarchy()
|
||||
nodes = telepathic._extract_semantic_nodes(xml)
|
||||
|
||||
|
||||
@@ -32,6 +32,20 @@ class CommentPlugin(BehaviorPlugin):
|
||||
if ctx.session_state.check_limit(SessionState.Limit.COMMENTS):
|
||||
return False
|
||||
|
||||
# Safety Guard: Do not comment on stories or grids
|
||||
xml_lower = (ctx.context_xml or "").lower()
|
||||
STORY_MARKERS = (
|
||||
"reel_viewer_media_layout",
|
||||
"reel_viewer_header",
|
||||
"reel_viewer_progress_bar",
|
||||
"reel_viewer_root",
|
||||
)
|
||||
if any(marker in xml_lower for marker in STORY_MARKERS):
|
||||
return False
|
||||
|
||||
if "explore_action_bar" in xml_lower or "profile_tabs_container" in xml_lower:
|
||||
return False
|
||||
|
||||
config = self.get_config(ctx)
|
||||
comment_pct = float(config.get("percentage", getattr(ctx.configs.args, "comment_percentage", 0))) / 100.0
|
||||
|
||||
@@ -78,7 +92,7 @@ class CommentPlugin(BehaviorPlugin):
|
||||
# 4. Type and post
|
||||
if nav_graph.do("type and post comment", text=text):
|
||||
logger.info(f"💬 [Comment] Posted to @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=False, liked=False)
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=False, scraped=False)
|
||||
ctx.session_state.totalComments += 1
|
||||
return BehaviorResult(executed=True, interactions=1, metadata={"text": text})
|
||||
|
||||
|
||||
@@ -40,7 +40,15 @@ class DarwinDwellPlugin(BehaviorPlugin):
|
||||
logger.info("🐢 [DarwinDwell] Executing organic dwell behaviors...")
|
||||
darwin.execute_micro_wobble(ctx.device)
|
||||
res_score = ctx.shared_state.get("res_score", 1.0)
|
||||
darwin.execute_proof_of_resonance(ctx.device, res_score)
|
||||
darwin.execute_proof_of_resonance(
|
||||
ctx.device,
|
||||
res_score,
|
||||
nav_graph=ctx.cognitive_stack.get("nav_graph"),
|
||||
configs=ctx.configs,
|
||||
resonance_oracle=ctx.cognitive_stack.get("oracle"),
|
||||
username=ctx.username,
|
||||
context_xml=ctx.context_xml or ctx.device.dump_hierarchy(),
|
||||
)
|
||||
else:
|
||||
logger.info("🐢 [DarwinDwell] Darwin engine missing. Falling back to static sleep.")
|
||||
sleep(2.5 * ctx.sleep_mod)
|
||||
|
||||
@@ -49,9 +49,9 @@ class FollowPlugin(BehaviorPlugin):
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if nav_graph.do("tap follow button"):
|
||||
if nav_graph.do("tap 'Follow' button") or nav_graph.do("tap 'Following' button"):
|
||||
logger.info(f"🤝 [Follow] Followed @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, liked=False)
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=True, scraped=False)
|
||||
|
||||
# Buffer for follow animations to close
|
||||
sleep(random.uniform(1.8, 3.2) * ctx.sleep_mod)
|
||||
|
||||
@@ -30,6 +30,7 @@ class LikePlugin(BehaviorPlugin):
|
||||
from GramAddict.core.session_state import SessionState
|
||||
|
||||
if ctx.session_state.check_limit(SessionState.Limit.LIKES):
|
||||
logger.error("LikePlugin: limit check failed")
|
||||
return False
|
||||
|
||||
config = self.get_config(ctx)
|
||||
@@ -54,7 +55,8 @@ class LikePlugin(BehaviorPlugin):
|
||||
|
||||
if nav_graph.do("tap like button"):
|
||||
logger.info(f"❤️ [Like] Liked post by @{ctx.username} ✓")
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=False, liked=True)
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=True, followed=False, scraped=False)
|
||||
ctx.session_state.totalLikes += 1
|
||||
return BehaviorResult(executed=True, interactions=1)
|
||||
|
||||
return BehaviorResult(executed=False)
|
||||
|
||||
@@ -3,7 +3,6 @@ from time import sleep
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext, BehaviorPlugin, BehaviorResult
|
||||
from GramAddict.core.diagnostic_dump import dump_ui_state
|
||||
from GramAddict.core.physics.humanized_input import humanized_scroll
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
@@ -43,10 +42,25 @@ class ObstacleGuardPlugin(BehaviorPlugin):
|
||||
|
||||
misses = ctx.shared_state.get("consecutive_marker_misses", 0)
|
||||
|
||||
# ── System Dialog / Permission Modal (e.g. "Allow Instagram to record audio?") ──
|
||||
if situation == SituationType.OBSTACLE_SYSTEM:
|
||||
logger.warning("⚠️ [ObstacleGuard] System permission dialog detected. Dismissing with BACK...")
|
||||
ctx.device.press("back")
|
||||
sleep(1.5 * ctx.sleep_mod)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
|
||||
# ── Foreign App Takeover (e.g. browser opened, wrong app in foreground) ──
|
||||
if situation == SituationType.OBSTACLE_FOREIGN_APP:
|
||||
logger.warning("⚠️ [ObstacleGuard] Foreign app detected. Pressing BACK to recover...")
|
||||
ctx.device.press("back")
|
||||
sleep(1.5 * ctx.sleep_mod)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
|
||||
# ── Instagram Modal / Overlay (survey, "Not Now" prompt, creation flow) ──
|
||||
if situation == SituationType.OBSTACLE_MODAL:
|
||||
if misses >= 2:
|
||||
logger.error("🛑 [ObstacleGuard] Failed to recover from OBSTACLE_MODAL after multiple attempts.")
|
||||
sae.unlearn_current_state()
|
||||
sae.unlearn_current_state(xml)
|
||||
dump_ui_state(ctx.device, f"fatal_obstacle_{ctx.session_state.job_target}")
|
||||
return BehaviorResult(executed=True, should_skip=True, metadata={"return_code": "CONTEXT_LOST"})
|
||||
|
||||
@@ -57,7 +71,7 @@ class ObstacleGuardPlugin(BehaviorPlugin):
|
||||
# Check recovery
|
||||
new_xml = ctx.device.dump_hierarchy()
|
||||
tele = TelepathicEngine.get_instance()
|
||||
best_node = tele.find_best_node(new_xml, intent_description="Dismiss obstacle")
|
||||
best_node = tele.find_best_node(new_xml, intent_description="Dismiss obstacle", device=ctx.device)
|
||||
if best_node:
|
||||
ctx.device.click(best_node.get("x", 0), best_node.get("y", 0))
|
||||
|
||||
@@ -69,16 +83,4 @@ class ObstacleGuardPlugin(BehaviorPlugin):
|
||||
|
||||
return BehaviorResult(executed=True, should_skip=True) # Restart loop for same post or next
|
||||
|
||||
else: # SituationType.NORMAL
|
||||
if "row_feed_button_like" not in xml:
|
||||
logger.info("🧩 [ObstacleGuard] Missing feed markers. Scrolling...")
|
||||
ctx.shared_state["consecutive_marker_misses"] = misses + 1
|
||||
if ctx.shared_state["consecutive_marker_misses"] >= 3:
|
||||
logger.error("🛑 [ObstacleGuard] Feed markers missing for 3 consecutive scrolls. Giving up.")
|
||||
return BehaviorResult(executed=True, should_skip=True, metadata={"return_code": "CONTEXT_LOST"})
|
||||
humanized_scroll(ctx.device)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
else:
|
||||
ctx.shared_state["consecutive_marker_misses"] = 0
|
||||
|
||||
return BehaviorResult(executed=False)
|
||||
|
||||
@@ -26,7 +26,17 @@ class PerfectSnappingPlugin(BehaviorPlugin):
|
||||
return 90
|
||||
|
||||
def can_activate(self, ctx: BehaviorContext) -> bool:
|
||||
return getattr(self, "_enabled", True)
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
# Perfect snapping is only for feed posts.
|
||||
# Do not snap if we are on a profile page, explore grid, or modal.
|
||||
from GramAddict.core.perception.feed_analysis import has_feed_markers
|
||||
|
||||
if not has_feed_markers(ctx.context_xml):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
aligned = _align_active_post(ctx.device)
|
||||
|
||||
@@ -26,15 +26,24 @@ class PostDataExtractionPlugin(BehaviorPlugin):
|
||||
return 85
|
||||
|
||||
def can_activate(self, ctx: BehaviorContext) -> bool:
|
||||
return getattr(self, "_enabled", True) and ctx.context_xml is not None
|
||||
from GramAddict.core.perception.feed_analysis import has_feed_markers
|
||||
|
||||
return getattr(self, "_enabled", True) and ctx.context_xml is not None and has_feed_markers(ctx.context_xml)
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
logger.debug("🧩 [PostDataExtraction] Extracting post metadata...")
|
||||
post_data = extract_post_content(ctx.context_xml)
|
||||
post_data = extract_post_content(ctx.context_xml, device=ctx.device)
|
||||
|
||||
if post_data:
|
||||
ctx.post_data = post_data
|
||||
ctx.username = post_data.get("username", "")
|
||||
|
||||
if post_data.get("username_missing") or not ctx.username:
|
||||
logger.error(
|
||||
"❌ [PostDataExtraction] FAILED: Post author username is empty or missing! Halting interaction."
|
||||
)
|
||||
return BehaviorResult(executed=False, metadata={"error": "Empty username extracted"})
|
||||
|
||||
logger.info(f"📝 [PostDataExtraction] Post by @{ctx.username} extracted.")
|
||||
return BehaviorResult(executed=True)
|
||||
|
||||
|
||||
@@ -50,8 +50,11 @@ class RepostPlugin(BehaviorPlugin):
|
||||
|
||||
nav_graph = QNavGraph(ctx.device)
|
||||
|
||||
if nav_graph.do("share to story"):
|
||||
logger.info(f"📤 [Repost] Shared post by @{ctx.username} to story ✓")
|
||||
return BehaviorResult(executed=True, interactions=1)
|
||||
# We must click the send post button first
|
||||
if nav_graph.do("tap send post button"):
|
||||
# A modal should appear, now click add to story
|
||||
if nav_graph.do("tap add to story"):
|
||||
logger.info(f"📤 [Repost] Shared post by @{ctx.username} to story ✓")
|
||||
return BehaviorResult(executed=True, interactions=1)
|
||||
|
||||
return BehaviorResult(executed=False)
|
||||
|
||||
@@ -49,11 +49,36 @@ class ResonanceEvaluatorPlugin(BehaviorPlugin):
|
||||
tele = ctx.cognitive_stack.get("telepathic")
|
||||
if tele:
|
||||
logger.info("✨ [Resonance] Performing visual vibe check...")
|
||||
vibe = tele.evaluate_post_vibe()
|
||||
vibe_score = vibe.get("quality_score", 5) / 10.0
|
||||
if vibe.get("matches_niche"):
|
||||
vibe_score = min(1.0, vibe_score + 0.2)
|
||||
res_score = (res_score * 0.3) + (vibe_score * 0.7)
|
||||
|
||||
# BUG 5 Fix: Read target_audience or persona_interests
|
||||
raw_interests = getattr(ctx.configs.args, "persona_interests", "")
|
||||
if not raw_interests:
|
||||
raw_interests = getattr(ctx.configs.args, "target_audience", "")
|
||||
|
||||
if isinstance(raw_interests, list):
|
||||
persona_interests = [str(i).strip() for i in raw_interests if str(i).strip()]
|
||||
else:
|
||||
persona_interests = [i.strip() for i in str(raw_interests).split(",") if i.strip()]
|
||||
|
||||
vibe = tele.evaluate_post_vibe(ctx.device, persona_interests)
|
||||
if vibe is None:
|
||||
logger.warning(
|
||||
"✨ [Resonance] VLM vibe check returned None (truncated JSON?). Keeping neutral score."
|
||||
)
|
||||
else:
|
||||
if vibe.get("is_ad"):
|
||||
logger.info("🛡️ [Resonance Oracle] Visually identified post as an Ad! Skipping...")
|
||||
marker = vibe.get("ad_marker_text")
|
||||
if marker and marker.strip():
|
||||
from GramAddict.core.utils import learn_ad_marker
|
||||
learn_ad_marker(marker, ctx.context_xml)
|
||||
humanized_scroll(ctx.device)
|
||||
return BehaviorResult(executed=True, should_skip=True)
|
||||
|
||||
# BUG 6 Fix: VLM returns {"should_like": true/false}, not "quality_score"
|
||||
should_like = vibe.get("should_like", False)
|
||||
vibe_score = 1.0 if should_like else 0.2
|
||||
res_score = (res_score * 0.3) + (vibe_score * 0.7)
|
||||
|
||||
ctx.shared_state["res_score"] = res_score
|
||||
logger.info(f"📊 [Resonance] Post Score: {res_score:.2f}")
|
||||
|
||||
78
GramAddict/core/behaviors/scrape_profile.py
Normal file
78
GramAddict/core/behaviors/scrape_profile.py
Normal file
@@ -0,0 +1,78 @@
|
||||
import logging
|
||||
|
||||
from GramAddict.core.behaviors import BehaviorContext, BehaviorPlugin, BehaviorResult
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScrapeProfilePlugin(BehaviorPlugin):
|
||||
"""
|
||||
Extracts profile metadata (followers, following, bio) when visiting a profile.
|
||||
|
||||
Priority: 45. (Runs after ProfileGuard, before deep interactions like GridLike)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._enabled = True
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "scrape_profile"
|
||||
|
||||
@property
|
||||
def priority(self) -> int:
|
||||
return 45
|
||||
|
||||
def can_activate(self, ctx: BehaviorContext) -> bool:
|
||||
if not getattr(self, "_enabled", True):
|
||||
return False
|
||||
|
||||
# Only activate if scrape_profiles is True in config
|
||||
if not getattr(ctx.configs.args, "scrape_profiles", False):
|
||||
return False
|
||||
|
||||
# Only activate when we are actively visiting a profile (via ProfileVisitPlugin)
|
||||
nav_graph = ctx.cognitive_stack.get("nav_graph")
|
||||
if not nav_graph or nav_graph.current_state != "ProfileView":
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def execute(self, ctx: BehaviorContext) -> BehaviorResult:
|
||||
from colorama import Fore
|
||||
|
||||
logger.info(f"📊 [Scraping] Extracting metadata for @{ctx.username}...", extra={"color": f"{Fore.CYAN}"})
|
||||
|
||||
telepathic = ctx.cognitive_stack.get("telepathic") or TelepathicEngine.get_instance()
|
||||
crm = ctx.cognitive_stack.get("crm")
|
||||
|
||||
xml_check = ctx.context_xml or ctx.device.dump_hierarchy()
|
||||
|
||||
f_node = telepathic.find_best_node(xml_check, "Followers count text or number", device=ctx.device)
|
||||
fg_node = telepathic.find_best_node(xml_check, "Following count text or number", device=ctx.device)
|
||||
bio_node = telepathic.find_best_node(xml_check, "User biography or description text", device=ctx.device)
|
||||
|
||||
scraped_data = {
|
||||
"username": ctx.username,
|
||||
"followers": f_node.get("text") if f_node else "unknown",
|
||||
"following": fg_node.get("text") if fg_node else "unknown",
|
||||
"bio": bio_node.get("text") if bio_node else "No bio",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"✅ [Scraping] Data acquired: {scraped_data['followers']} followers, {scraped_data['following']} following."
|
||||
)
|
||||
|
||||
ctx.session_state.add_interaction(source=ctx.username, succeed=False, followed=False, scraped=True)
|
||||
|
||||
if crm:
|
||||
try:
|
||||
crm.enrich_lead(ctx.username, scraped_data)
|
||||
logger.info(f"💾 [CRM] Enriched lead @{ctx.username} in database.")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ [CRM] Failed to enrich lead @{ctx.username}: {e}")
|
||||
|
||||
# Return executed=True, but we don't return interactions=1 since it's just data extraction
|
||||
return BehaviorResult(executed=True)
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
try:
|
||||
import psutil
|
||||
@@ -9,21 +10,6 @@ except ImportError:
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
|
||||
|
||||
def log_metabolic_rate():
|
||||
if psutil is None:
|
||||
logging.getLogger(__name__).debug("🧬 [Metabolism] psutil not installed. Skipping memory log.")
|
||||
return
|
||||
try:
|
||||
process = psutil.Process(os.getpid())
|
||||
mem_info = process.memory_info()
|
||||
logging.getLogger(__name__).info(
|
||||
f"🧬 [Metabolism] RSS: {mem_info.rss / 1024 / 1024:.2f} MB | VMS: {mem_info.vms / 1024 / 1024:.2f} MB"
|
||||
)
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).debug(f"🧬 [Metabolism] Failed to log memory: {e}")
|
||||
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
from GramAddict.core.account_switcher import verify_and_switch_account
|
||||
@@ -66,8 +52,6 @@ from GramAddict.core.physics.timing import (
|
||||
wait_for_story_loaded as _wait_for_story_loaded_impl,
|
||||
)
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.qdrant_memory import ParasocialCRMDB
|
||||
from GramAddict.core.resonance_engine import ResonanceEngine
|
||||
from GramAddict.core.sensors.honeypot_radome import HoneypotRadome
|
||||
from GramAddict.core.session_state import SessionState, SessionStateEncoder
|
||||
from GramAddict.core.swarm_protocol import SwarmProtocol
|
||||
@@ -84,6 +68,21 @@ from GramAddict.core.utils import (
|
||||
)
|
||||
from GramAddict.core.zero_latency_engine import ZeroLatencyEngine
|
||||
|
||||
|
||||
def log_metabolic_rate():
|
||||
if psutil is None:
|
||||
logging.getLogger(__name__).debug("🧬 [Metabolism] psutil not installed. Skipping memory log.")
|
||||
return
|
||||
try:
|
||||
process = psutil.Process(os.getpid())
|
||||
mem_info = process.memory_info()
|
||||
logging.getLogger(__name__).info(
|
||||
f"🧬 [Metabolism] RSS: {mem_info.rss / 1024 / 1024:.2f} MB | VMS: {mem_info.vms / 1024 / 1024:.2f} MB"
|
||||
)
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).debug(f"🧬 [Metabolism] Failed to log memory: {e}")
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -94,9 +93,8 @@ def check_production_integrity():
|
||||
"""
|
||||
import sys
|
||||
|
||||
# If we are in a pytest session, we expect and allow mocks
|
||||
if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
|
||||
return
|
||||
# We no longer skip this in tests. Production integrity must hold everywhere.
|
||||
pass
|
||||
|
||||
try:
|
||||
from unittest.mock import MagicMock
|
||||
@@ -178,13 +176,27 @@ def start_bot(**kwargs):
|
||||
)
|
||||
persona_interests = [p.strip() for p in persona_raw.split(",") if p.strip()] if persona_raw else []
|
||||
|
||||
global_goal = getattr(configs.args, "goal", None)
|
||||
if global_goal:
|
||||
persona_interests.insert(0, global_goal)
|
||||
logger.info(
|
||||
f"🎯 [Autonomous Directive] Overriding target audience with high-level goal: {global_goal}",
|
||||
extra={"color": f"{Style.BRIGHT}{Fore.GREEN}"},
|
||||
)
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
from GramAddict.core.interaction import LLMWriter
|
||||
from GramAddict.core.qdrant_memory import DMMemoryDB, ParasocialCRMDB
|
||||
from GramAddict.core.resonance_engine import ResonanceEngine
|
||||
|
||||
dopamine = DopamineEngine()
|
||||
crm_db = ParasocialCRMDB()
|
||||
dm_memory_db = DMMemoryDB()
|
||||
resonance_oracle = ResonanceEngine(username, persona_interests=persona_interests, crm=crm_db)
|
||||
writer = LLMWriter(username, persona_interests, configs)
|
||||
active_inference = ActiveInferenceEngine(username)
|
||||
|
||||
# Core Autonomous Engines
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
GoalExecutor.get_instance(device, username)
|
||||
zero_engine = ZeroLatencyEngine(device)
|
||||
@@ -235,6 +247,8 @@ def start_bot(**kwargs):
|
||||
"telepathic": telepathic,
|
||||
"darwin": darwin,
|
||||
"crm": crm_db,
|
||||
"dm_memory": dm_memory_db,
|
||||
"writer": writer,
|
||||
}
|
||||
|
||||
from GramAddict.core.behaviors import PluginRegistry
|
||||
@@ -256,6 +270,7 @@ def start_bot(**kwargs):
|
||||
from GramAddict.core.behaviors.rabbit_hole import RabbitHolePlugin
|
||||
from GramAddict.core.behaviors.repost import RepostPlugin
|
||||
from GramAddict.core.behaviors.resonance_evaluator import ResonanceEvaluatorPlugin
|
||||
from GramAddict.core.behaviors.scrape_profile import ScrapeProfilePlugin
|
||||
from GramAddict.core.behaviors.story_view import StoryViewPlugin
|
||||
|
||||
PluginRegistry.reset()
|
||||
@@ -279,6 +294,7 @@ def start_bot(**kwargs):
|
||||
plugin_registry.register(CommentPlugin())
|
||||
plugin_registry.register(RepostPlugin())
|
||||
plugin_registry.register(PostInteractionPlugin())
|
||||
plugin_registry.register(ScrapeProfilePlugin())
|
||||
|
||||
cognitive_stack["plugin_registry"] = plugin_registry
|
||||
|
||||
@@ -290,7 +306,26 @@ def start_bot(**kwargs):
|
||||
cognitive_stack["dojo"] = dojo
|
||||
|
||||
try:
|
||||
bot_start_time = datetime.now()
|
||||
max_runtime = getattr(configs.args, "max_runtime_minutes", None)
|
||||
|
||||
dopamine.global_start_time = bot_start_time
|
||||
dopamine.global_max_runtime_minutes = max_runtime
|
||||
GoalExecutor.global_start_time = bot_start_time
|
||||
GoalExecutor.global_max_runtime_minutes = max_runtime
|
||||
|
||||
while True:
|
||||
if max_runtime:
|
||||
from datetime import timedelta
|
||||
|
||||
elapsed = datetime.now() - bot_start_time
|
||||
if elapsed > timedelta(minutes=max_runtime):
|
||||
logger.info(
|
||||
f"🛑 [Timeout] Maximum runtime of {max_runtime} minutes reached. Stopping bot.",
|
||||
extra={"color": f"{Fore.RED}"},
|
||||
)
|
||||
break
|
||||
|
||||
set_time_delta(configs.args)
|
||||
inside_working_hours, time_left = SessionState.inside_working_hours(
|
||||
configs.args.working_hours, configs.args.time_delta_session
|
||||
@@ -341,9 +376,7 @@ def start_bot(**kwargs):
|
||||
logger.info(
|
||||
f"🧠 [Agent Orchestrator] Session started. Strategy: {growth_brain.strategy} | Persona: {getattr(configs.args, 'agent_persona', 'unknown')}"
|
||||
)
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor
|
||||
|
||||
# 1. Starten wir den GOAP Executor, um die UI-Struktur autonom zu erfassen
|
||||
goap = GoalExecutor.get_instance(device, username)
|
||||
|
||||
# --- PHASE 0: Autonomous Profile Scanning ---
|
||||
@@ -439,31 +472,68 @@ def start_bot(**kwargs):
|
||||
has_scanned_own_profile = True
|
||||
|
||||
while not dopamine.is_app_session_over():
|
||||
# 1. Ask the Growth Brain for a Desire
|
||||
current_desire = growth_brain.get_current_desire(dopamine)
|
||||
# ── 1. Generate available tasks from mission + plugins ──
|
||||
from GramAddict.core.goal_decomposer import GoalDecomposer
|
||||
|
||||
if current_desire == "ShiftContext":
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart to clear context.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
decomposer = GoalDecomposer(
|
||||
plugins=configs.config.get("plugins", {}) if configs.config else {},
|
||||
actions={
|
||||
k: getattr(configs.args, k, None)
|
||||
for k in ("feed", "explore", "reels")
|
||||
if getattr(configs.args, k, None)
|
||||
},
|
||||
mission=configs.config.get("mission", {}) if configs.config else {},
|
||||
)
|
||||
available_tasks = decomposer.generate_tasks()
|
||||
|
||||
# 2. Map Desire to Sub-Feed
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList", "MessageInbox"],
|
||||
}
|
||||
if not available_tasks:
|
||||
# No plugins enabled = nothing to do. Fall back to legacy desire system.
|
||||
current_desire = growth_brain.get_current_desire(dopamine)
|
||||
if current_desire == "ShiftContext":
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
|
||||
import secrets
|
||||
# Legacy desire → target mapping (kept for backward compatibility)
|
||||
target_map = {
|
||||
"DiscoverNewContent": ["ExploreFeed", "ReelsFeed"],
|
||||
"NurtureCommunity": ["HomeFeed", "StoriesFeed"],
|
||||
"SocialReciprocity": ["FollowingList"],
|
||||
}
|
||||
|
||||
options = target_map.get(current_desire, ["HomeFeed"])
|
||||
current_target = secrets.choice(options)
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
target_map["SocialReciprocity"].append("MessageInbox")
|
||||
|
||||
logger.info(f"🧠 [Agent Orchestrator] Desire '{current_desire}' -> Routed to {current_target}")
|
||||
import secrets
|
||||
|
||||
options = target_map.get(current_desire, ["HomeFeed"])
|
||||
current_target = secrets.choice(options)
|
||||
else:
|
||||
# ── 2. Select a concrete Task ──
|
||||
selected_task = growth_brain.select_task(dopamine, available_tasks)
|
||||
|
||||
if selected_task is None:
|
||||
# ShiftContext signal from high boredom
|
||||
logger.info("🧠 [Free Will] Boredom critical. Forcing app restart to clear context.")
|
||||
device.app_stop(device.app_id)
|
||||
random_sleep(2.0, 4.0)
|
||||
device.app_start(device.app_id, use_monkey=True)
|
||||
random_sleep(4.0, 6.0)
|
||||
dopamine.boredom = max(0.0, dopamine.boredom * 0.2)
|
||||
continue
|
||||
|
||||
current_target = selected_task.target_screen
|
||||
logger.info(
|
||||
f"🎯 [GoalDecomposer] Task: {selected_task.intent} "
|
||||
f"→ {current_target} (budget={selected_task.budget_posts})"
|
||||
)
|
||||
|
||||
logger.info(f"🧠 [Agent Orchestrator] Routed to {current_target}")
|
||||
|
||||
logger.info(f"⚡ Navigating to {current_target}")
|
||||
success = nav_graph.navigate_to(current_target, zero_engine)
|
||||
@@ -488,7 +558,9 @@ def start_bot(**kwargs):
|
||||
continue
|
||||
elif current_target == "StoriesFeed":
|
||||
logger.info("📱 Locating story tray on HomeFeed...")
|
||||
nav_graph.do("tap story ring avatar")
|
||||
if not nav_graph.do("tap story ring avatar"):
|
||||
logger.warning("❌ Failed to tap story ring avatar. Retrying next loop.")
|
||||
continue
|
||||
post_loaded = _wait_for_story_loaded(device, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning("❌ Stories failed to open from HomeFeed. Retrying next loop.")
|
||||
@@ -613,6 +685,7 @@ def _interact_with_profile(device, configs, username, session_state, sleep_mod,
|
||||
|
||||
if cognitive_stack is None:
|
||||
cognitive_stack = {}
|
||||
_validate_cognitive_stack(cognitive_stack, "ProfileInteraction")
|
||||
|
||||
if hasattr(session_state, "my_username") and username == session_state.my_username:
|
||||
logger.info(f"🤝 [Deep Interaction] Skipping own profile @{username} to prevent self-interactions.")
|
||||
@@ -752,6 +825,7 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
|
||||
|
||||
from colorama import Fore
|
||||
|
||||
_validate_cognitive_stack(cognitive_stack, "StoriesLoop")
|
||||
logger.info("🎬 [StoriesFeed] Starting native story binging loop...", extra={"color": f"{Fore.CYAN}"})
|
||||
|
||||
dopamine = cognitive_stack.get("dopamine")
|
||||
@@ -786,6 +860,20 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
|
||||
logger.warning("Failed to dump UI hierarchy in StoriesFeed.")
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
# ── Perimeter Guard: Verify we're still inside Instagram ──
|
||||
# Production bug 2026-05-03: A story's swipe-up link opened the Play Store,
|
||||
# and the loop kept tapping blindly on com.android.vending for 5+ iterations.
|
||||
packages = set(re.findall(r'package="([^"]+)"', xml_dump))
|
||||
app_id = getattr(device, "app_id", "com.instagram.android")
|
||||
if packages and app_id not in packages:
|
||||
logger.error(
|
||||
f"🚨 [StoriesFeed] FOREIGN APP DETECTED! Packages: {packages}. "
|
||||
f"A story link likely opened an external app. Aborting loop."
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
if getattr(configs.args, "ignore_close_friends", False):
|
||||
if "enge freunde" in xml_dump.lower() or "close friend" in xml_dump.lower():
|
||||
logger.info(
|
||||
@@ -807,6 +895,36 @@ def _run_zero_latency_stories_loop(device, configs, session_state, cognitive_sta
|
||||
return "FEED_EXHAUSTED"
|
||||
|
||||
|
||||
def _validate_cognitive_stack(cognitive_stack, context_name):
|
||||
"""
|
||||
Validates that the cognitive stack has all required engine dependencies
|
||||
injected properly. This is the ultimate zero-trust guard for plugins.
|
||||
"""
|
||||
if not isinstance(cognitive_stack, dict):
|
||||
raise TypeError(f"[{context_name}] CognitiveStack must be a dict, got {type(cognitive_stack)}")
|
||||
|
||||
required_engines = [
|
||||
"dopamine",
|
||||
"darwin",
|
||||
"resonance",
|
||||
"active_inference",
|
||||
"growth_brain",
|
||||
"swarm",
|
||||
"writer",
|
||||
"nav_graph",
|
||||
"zero_engine",
|
||||
"telepathic",
|
||||
]
|
||||
|
||||
missing = [eng for eng in required_engines if eng not in cognitive_stack or cognitive_stack[eng] is None]
|
||||
if missing:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(f"🚨 [{context_name}] CognitiveStack missing required engines: {missing}")
|
||||
raise ValueError(f"[{context_name}] CognitiveStack missing required engines: {missing}")
|
||||
|
||||
|
||||
def _run_zero_latency_feed_loop(
|
||||
device, zero_engine, nav_graph, configs, session_state, job_target, cognitive_stack, is_reels=False
|
||||
):
|
||||
@@ -820,6 +938,7 @@ def _run_zero_latency_feed_loop(
|
||||
- Darwin is the SOLE dwell controller → no duplicate sleep calls
|
||||
- SwarmProtocol emits pheromones after successful interactions
|
||||
"""
|
||||
_validate_cognitive_stack(cognitive_stack, "FeedLoop")
|
||||
logger.info(f"🔄 Entering Zero-Latency Interaction Pool. Feed: {job_target}")
|
||||
|
||||
dopamine = cognitive_stack.get("dopamine")
|
||||
@@ -855,7 +974,19 @@ def _run_zero_latency_feed_loop(
|
||||
|
||||
elif governance_decision == "CHECK_CURIOSITY":
|
||||
logger.info("👀 [Curiosity] Spontaneously checking DMs / Notifications...")
|
||||
explore_target = random.choice(["MessageInbox", "Notifications"])
|
||||
|
||||
# 🛡️ Structural Guard: Curiosity targets (DMs, Notifications) are ONLY available on HomeFeed.
|
||||
# We must navigate there first, breaking current context.
|
||||
if not nav_graph.navigate_to("HomeFeed", zero_engine):
|
||||
logger.warning("❌ [Curiosity] Failed to navigate to HomeFeed. Aborting curiosity check.")
|
||||
continue
|
||||
sleep(random.uniform(1.0, 2.5))
|
||||
|
||||
dm_config = configs.get_plugin_config("dm_reply")
|
||||
if dm_config.get("enabled", False):
|
||||
explore_target = random.choice(["MessageInbox", "Notifications"])
|
||||
else:
|
||||
explore_target = "Notifications"
|
||||
|
||||
if explore_target == "MessageInbox":
|
||||
nav_graph.do("tap direct message icon inbox")
|
||||
@@ -887,6 +1018,17 @@ def _run_zero_latency_feed_loop(
|
||||
if cognitive_stack.get("radome"):
|
||||
context_xml = cognitive_stack.get("radome").sanitize_xml(context_xml)
|
||||
|
||||
# ── Perimeter Guard: Verify we're still inside Instagram ──
|
||||
# Parity with story loop guard (production bug 2026-05-03).
|
||||
if context_xml:
|
||||
feed_packages = set(re.findall(r'package="([^"]+)"', context_xml))
|
||||
feed_app_id = getattr(device, "app_id", "com.instagram.android")
|
||||
if feed_packages and feed_app_id not in feed_packages:
|
||||
logger.error(f"🚨 [FeedLoop] FOREIGN APP DETECTED! Packages: {feed_packages}. Aborting loop.")
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
# ── Execute Plugin Registry Behaviors (Feed Level) ──
|
||||
from GramAddict.core.behaviors import BehaviorContext, PluginRegistry
|
||||
|
||||
@@ -946,6 +1088,7 @@ def _run_zero_latency_search_loop(
|
||||
"""
|
||||
Executes the autonomous Search & Interact logic.
|
||||
"""
|
||||
_validate_cognitive_stack(cognitive_stack, "SearchLoop")
|
||||
logger.info("🧠 [Search Engine] Initiating keyword discovery...", extra={"color": f"{Style.BRIGHT}{Fore.CYAN}"})
|
||||
|
||||
import random
|
||||
@@ -971,6 +1114,16 @@ def _run_zero_latency_search_loop(
|
||||
xml = device.dump_hierarchy()
|
||||
telepathic = cognitive_stack.get("telepathic")
|
||||
|
||||
# ── Perimeter Guard: Verify we're still inside Instagram ──
|
||||
if xml:
|
||||
search_packages = set(re.findall(r'package="([^"]+)"', xml))
|
||||
search_app_id = getattr(device, "app_id", "com.instagram.android")
|
||||
if search_packages and search_app_id not in search_packages:
|
||||
logger.error(f"🚨 [SearchLoop] FOREIGN APP DETECTED! Packages: {search_packages}. Aborting loop.")
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
# Find search bar
|
||||
search_bar = telepathic.find_best_node(xml, "Search edit text box or magnifying glass input", device=device)
|
||||
if search_bar:
|
||||
|
||||
@@ -17,14 +17,12 @@ class Config:
|
||||
self.args = kwargs
|
||||
self.module = True
|
||||
else:
|
||||
# Avoid parsing sys.argv if we are running in a test environment (pytest)
|
||||
# as pytest arguments will cause argparse to fail with SystemExit: 2
|
||||
is_pytest = "pytest" in sys.modules
|
||||
if is_pytest:
|
||||
self.args = []
|
||||
else:
|
||||
self.args = sys.argv
|
||||
self.args = list(sys.argv)
|
||||
self.module = False
|
||||
|
||||
if not self.module and "--config" not in self.args:
|
||||
if os.path.exists("config.yml"):
|
||||
self.args.extend(["--config", "config.yml"])
|
||||
self.config = None
|
||||
self.config_list = None
|
||||
self.actions = {}
|
||||
@@ -81,6 +79,9 @@ class Config:
|
||||
self.username = self.username[0]
|
||||
self.debug = self.config.get("debug", False)
|
||||
self.app_id = self.config.get("app_id", "com.instagram.android")
|
||||
|
||||
# Autonomous goals removed — the bot now derives tasks from mission + plugins
|
||||
# via GoalDecomposer. See GramAddict/core/goal_decomposer.py.
|
||||
else:
|
||||
if "--debug" in self.args:
|
||||
self.debug = True
|
||||
@@ -137,12 +138,22 @@ class Config:
|
||||
self.parser.add_argument("--total-sessions", help="Total amount of sessions", default="-1")
|
||||
self.parser.add_argument("--working-hours", help="Working hours", default=None)
|
||||
self.parser.add_argument("--time-delta-session", help="Time delta between sessions", default=None)
|
||||
self.parser.add_argument(
|
||||
"--max-runtime-minutes", type=int, help="Maximum runtime in minutes before bot auto-exits", default=None
|
||||
)
|
||||
self.parser.add_argument("--restart-atx-agent", action="store_true", help="Restart atx agent")
|
||||
self.parser.add_argument("--allow-untested-ig-version", action="store_true", help="Allow untested IG version")
|
||||
self.parser.add_argument(
|
||||
"--blank-start",
|
||||
action="store_true",
|
||||
help="Wipe all learned navigation and telepathic memories on boot to start 100% blank.",
|
||||
help="Wipe all learned navigation and telepathic memories on boot to start 100%% blank.",
|
||||
)
|
||||
|
||||
self.parser.add_argument(
|
||||
"--goal",
|
||||
type=str,
|
||||
help="High-level autonomous goal for the bot (Tesla-style). Overrides config.yml goals.",
|
||||
default=None,
|
||||
)
|
||||
|
||||
# Interaction settings
|
||||
@@ -308,7 +319,7 @@ class Config:
|
||||
logger.debug(f"Arguments used: {' '.join(sys.argv[1:])}")
|
||||
if self.config:
|
||||
logger.debug(f"Config used: {self.config}")
|
||||
if len(sys.argv) <= 1:
|
||||
if len(sys.argv) <= 1 and not self.config:
|
||||
self.parser.print_help()
|
||||
exit(0)
|
||||
if self.config:
|
||||
|
||||
@@ -84,7 +84,6 @@ class DarwinEngine(QdrantBase):
|
||||
resonance: float,
|
||||
text_length: int = 0,
|
||||
nav_graph=None,
|
||||
zero_engine=None,
|
||||
configs=None,
|
||||
resonance_oracle=None,
|
||||
username=None,
|
||||
@@ -142,12 +141,22 @@ class DarwinEngine(QdrantBase):
|
||||
cy = h // 2
|
||||
|
||||
dur_ms = int(random.uniform(200, 500))
|
||||
device.shell(f"input swipe {int(cx)} {int(cy)} {int(cx + noise_x)} {int(cy + slip_distance)} {dur_ms}")
|
||||
|
||||
# Use physics-based injector instead of algorithmic 'input swipe'
|
||||
body = PhysicsBody.get_session_instance(device)
|
||||
injector = SendEventInjector.get_instance(device)
|
||||
start_pt = (int(cx), int(cy))
|
||||
end_pt = (int(cx + noise_x), int(cy + slip_distance))
|
||||
|
||||
points = BezierGesture.scroll_curve(start_pt, end_pt, body, n_points=5)
|
||||
timing = BezierGesture.compute_sigmoid_timing(len(points), dur_ms)
|
||||
injector.inject_gesture(points, timing, touch_major=body.get_touch_major())
|
||||
|
||||
time.sleep(random.uniform(0.5, 1.2))
|
||||
|
||||
# 4. Comment depth simulation (probabilistic & resonance-correlated)
|
||||
if profile["comment_read_dwell"] > 1.0 and resonance > 0.4 and random.random() < 0.3:
|
||||
if nav_graph and zero_engine:
|
||||
if nav_graph:
|
||||
if not self._has_comments(context_xml):
|
||||
logger.debug(" -> 🚫 [Darwin Engine] Skipping comment depth simulation (Post has 0 comments).")
|
||||
else:
|
||||
@@ -334,27 +343,34 @@ class DarwinEngine(QdrantBase):
|
||||
"""
|
||||
Heuristic to check if a post actually has comments to read.
|
||||
If it has 0 comments, checking them is suspicious bot behavior.
|
||||
|
||||
Zero-Maintenance: Uses only English text and resource_id patterns.
|
||||
Resource IDs are locale-invariant. English text in content_desc
|
||||
is used by Instagram internally and is reliable.
|
||||
"""
|
||||
low_xml = xml_string.lower()
|
||||
|
||||
# 1. Explicit zero comments checks
|
||||
if re.search(r"\b0\s*kommentare?\b", low_xml) or re.search(r"\b0\s*comment(?:s)?\b", low_xml):
|
||||
# 1. Explicit zero comments check (resource_id based + English fallback)
|
||||
if re.search(r"\b0\s*comment(?:s)?\b", low_xml):
|
||||
return False
|
||||
|
||||
# 2. Check for "view all" or similar prominent comment link texts
|
||||
if "view all" in low_xml or ("alle " in low_xml and "kommentare ansehen" in low_xml):
|
||||
if "view all" in low_xml:
|
||||
return True
|
||||
if "view 1 comment" in low_xml or "1 kommentar ansehen" in low_xml:
|
||||
if "view 1 comment" in low_xml:
|
||||
return True
|
||||
if "comment number is" in low_xml:
|
||||
return True
|
||||
|
||||
# 3. Check for specific counter elements > 0 in content descriptors
|
||||
# e.g. "by username, 23 comments" or "1,234 comments"
|
||||
has_number_of_comments = re.search(r"\b([1-9][0-9.,]*)\s*(?:comment(?:s)?|kommentare?)\b", low_xml)
|
||||
# 3. Structural: comment_textview_layout is present with a count > 0
|
||||
has_number_of_comments = re.search(r"\b([1-9][0-9.,]*)\s*comment(?:s)?\b", low_xml)
|
||||
if has_number_of_comments:
|
||||
return True
|
||||
|
||||
# 4. Structural: The comment button resource_id exists and has content
|
||||
if "row_feed_comment_textview_layout" in low_xml:
|
||||
return True
|
||||
|
||||
# If no indicators are found, assume the post has 0 comments.
|
||||
# The comment button exists, but there are no comments to read.
|
||||
return False
|
||||
|
||||
|
||||
@@ -36,15 +36,38 @@ def create_device(device_id, app_id, args=None):
|
||||
try:
|
||||
return DeviceFacade(device_id, app_id, args)
|
||||
except Exception as e:
|
||||
str(e)
|
||||
err_msg = str(e)
|
||||
err_type = str(type(e))
|
||||
if (
|
||||
"ConnectError" in err_type
|
||||
or "ConnectionRefusedError" in err_type
|
||||
or "ConnectionError" in err_type
|
||||
or "Timeout" in err_type
|
||||
if any(
|
||||
keyword in err_type or keyword in err_msg
|
||||
for keyword in ["ConnectError", "ConnectionRefused", "ConnectionError", "Timeout"]
|
||||
):
|
||||
logger.error(f"⚠️ [ADB ConnectError] Could not connect to device '{device_id}'.")
|
||||
|
||||
# Proactive Discovery
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(["adb", "devices"], capture_output=True, text=True, timeout=2)
|
||||
lines = [
|
||||
line.strip()
|
||||
for line in result.stdout.split("\n")
|
||||
if line.strip() and not line.startswith("List of devices")
|
||||
]
|
||||
devices = [line.split("\t")[0] for line in lines if "device" in line]
|
||||
|
||||
if devices:
|
||||
logger.info("🔍 Proactive Discovery: I found the following devices connected:")
|
||||
for d in devices:
|
||||
if d.split(":")[0] == device_id.split(":")[0]:
|
||||
logger.info(f" 👉 {d} (MATCHING IP - Is this the same device with a different port?)")
|
||||
else:
|
||||
logger.info(f" - {d}")
|
||||
else:
|
||||
logger.warning("🔍 Proactive Discovery: No ADB devices found. Is your phone authorized?")
|
||||
except Exception as discovery_err:
|
||||
logger.debug(f"Proactive discovery failed: {discovery_err}")
|
||||
|
||||
logger.error("👉 Please verify:")
|
||||
logger.error(" 1. Your phone is connected via USB or Wi-Fi.")
|
||||
logger.error(" 2. 'USB Debugging' is enabled in Developer Options.")
|
||||
@@ -158,6 +181,10 @@ class DeviceFacade:
|
||||
def press(self, key):
|
||||
self.deviceV2.press(key)
|
||||
|
||||
@adb_retry()
|
||||
def back(self):
|
||||
self.deviceV2.press("back")
|
||||
|
||||
@adb_retry()
|
||||
def click(self, x=None, y=None, obj=None):
|
||||
if obj:
|
||||
@@ -299,19 +326,51 @@ class DeviceFacade:
|
||||
xml = self.deviceV2.dump_hierarchy(compressed=True)
|
||||
|
||||
# Continuous Session Tracing
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
traces_root = os.path.join("debug", "session_traces")
|
||||
if not hasattr(self, "_trace_counter"):
|
||||
self._trace_counter = 0
|
||||
ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
self._trace_dir = os.path.join("debug", "session_traces", ts)
|
||||
self._trace_dir = os.path.join(traces_root, ts)
|
||||
os.makedirs(self._trace_dir, exist_ok=True)
|
||||
|
||||
# Cleanup: keep only last 5 session folders
|
||||
try:
|
||||
if os.path.exists(traces_root):
|
||||
folders = [
|
||||
os.path.join(traces_root, d)
|
||||
for d in os.listdir(traces_root)
|
||||
if os.path.isdir(os.path.join(traces_root, d))
|
||||
]
|
||||
folders.sort(key=os.path.getmtime)
|
||||
while len(folders) > 5:
|
||||
oldest = folders.pop(0)
|
||||
shutil.rmtree(oldest, ignore_errors=True)
|
||||
logger.info(f"🧹 [Cleanup] Removed old session trace: {oldest}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to cleanup old traces: {e}")
|
||||
|
||||
self._trace_counter += 1
|
||||
trace_path = os.path.join(self._trace_dir, f"{self._trace_counter:05d}.xml")
|
||||
with open(trace_path, "w", encoding="utf-8") as f:
|
||||
f.write(xml)
|
||||
|
||||
# Dump screenshot as well
|
||||
try:
|
||||
import base64
|
||||
|
||||
screenshot_b64 = self.get_screenshot_b64()
|
||||
if screenshot_b64:
|
||||
screenshot_data = base64.b64decode(screenshot_b64)
|
||||
screenshot_path = trace_path.replace(".xml", ".jpg")
|
||||
with open(screenshot_path, "wb") as f:
|
||||
f.write(screenshot_data)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to capture screenshot for session trace: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to write session trace: {e}")
|
||||
|
||||
@@ -323,6 +382,8 @@ class DeviceFacade:
|
||||
from io import BytesIO
|
||||
|
||||
img = self.deviceV2.screenshot()
|
||||
if img is None:
|
||||
return None
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="JPEG", quality=70) # Compressed for target latency
|
||||
return base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
@@ -18,19 +18,12 @@ from datetime import datetime
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DUMP_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "debug", "xml_dumps")
|
||||
MAX_DUMPS_PER_CATEGORY = 50
|
||||
MAX_DUMPS_PER_CATEGORY = 5
|
||||
|
||||
|
||||
def dump_ui_state(device, reason: str, extra_context: dict = None):
|
||||
"""
|
||||
Capture and save the current UI hierarchy to disk for debugging.
|
||||
|
||||
Args:
|
||||
device: The uiautomator2 device facade.
|
||||
reason: Short tag for the failure type. Used for filename grouping.
|
||||
Examples: 'context_lost', 'vlm_hallucination', 'nav_failure',
|
||||
'stuck_on_post', 'unexpected_screen'
|
||||
extra_context: Optional dict with additional metadata (intent, expected state, etc.)
|
||||
Capture and save the current UI hierarchy and screenshot to disk for debugging.
|
||||
"""
|
||||
try:
|
||||
os.makedirs(DUMP_DIR, exist_ok=True)
|
||||
@@ -48,11 +41,25 @@ def dump_ui_state(device, reason: str, extra_context: dict = None):
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
f.write(xml)
|
||||
|
||||
# Capture and write screenshot
|
||||
try:
|
||||
import base64
|
||||
|
||||
screenshot_b64 = device.get_screenshot_b64()
|
||||
if screenshot_b64:
|
||||
screenshot_data = base64.b64decode(screenshot_b64)
|
||||
screenshot_path = filepath.replace(".xml", ".jpg")
|
||||
with open(screenshot_path, "wb") as f:
|
||||
f.write(screenshot_data)
|
||||
except Exception as e:
|
||||
logger.debug(f"[Diagnostic] Could not capture screenshot: {e}")
|
||||
|
||||
# Write companion metadata JSON
|
||||
meta = {
|
||||
"reason": reason,
|
||||
"timestamp": ts,
|
||||
"xml_file": filename,
|
||||
"screenshot_file": filename.replace(".xml", ".jpg"),
|
||||
}
|
||||
# Capture the session log if available
|
||||
try:
|
||||
@@ -77,7 +84,7 @@ def dump_ui_state(device, reason: str, extra_context: dict = None):
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"📸 [Diagnostic] UI state and session log dumped for '{reason}': {filepath}")
|
||||
logger.info(f"📸 [Diagnostic] UI state, screenshot, and session log dumped for '{reason}': {filepath}")
|
||||
|
||||
# Rotate old dumps for this category
|
||||
_rotate_dumps(safe_reason)
|
||||
@@ -90,18 +97,50 @@ def dump_ui_state(device, reason: str, extra_context: dict = None):
|
||||
return None
|
||||
|
||||
|
||||
def _rotate_dumps(category_prefix: str):
|
||||
"""Keep only the last MAX_DUMPS_PER_CATEGORY dumps per category."""
|
||||
def _rotate_dumps(category_prefix: str = None):
|
||||
"""Keep only the last MAX_DUMPS_PER_CATEGORY dumps per category. If no category, cleans all."""
|
||||
try:
|
||||
all_files = sorted([f for f in os.listdir(DUMP_DIR) if f.startswith(category_prefix) and f.endswith(".xml")])
|
||||
if not os.path.exists(DUMP_DIR):
|
||||
return
|
||||
|
||||
if len(all_files) > MAX_DUMPS_PER_CATEGORY:
|
||||
files_to_remove = all_files[: len(all_files) - MAX_DUMPS_PER_CATEGORY]
|
||||
for f in files_to_remove:
|
||||
xml_path = os.path.join(DUMP_DIR, f)
|
||||
meta_path = xml_path.replace(".xml", ".meta.json")
|
||||
os.remove(xml_path)
|
||||
if os.path.exists(meta_path):
|
||||
os.remove(meta_path)
|
||||
except Exception:
|
||||
pass
|
||||
# Get all unique timestamps/prefixes
|
||||
all_files = os.listdir(DUMP_DIR)
|
||||
prefixes = set()
|
||||
for f in all_files:
|
||||
# Format is usually reason__timestamp.ext
|
||||
if "__" in f:
|
||||
prefix = f.split(".")[0]
|
||||
prefixes.add(prefix)
|
||||
|
||||
# Group prefixes by category
|
||||
categories = {}
|
||||
for p in prefixes:
|
||||
parts = p.split("__")
|
||||
if len(parts) >= 2:
|
||||
cat = parts[0]
|
||||
if cat not in categories:
|
||||
categories[cat] = []
|
||||
categories[cat].append(p)
|
||||
|
||||
for cat, prefs in categories.items():
|
||||
if category_prefix and cat != category_prefix:
|
||||
continue
|
||||
|
||||
prefs.sort() # chronological
|
||||
if len(prefs) > MAX_DUMPS_PER_CATEGORY:
|
||||
prefs_to_remove = prefs[: len(prefs) - MAX_DUMPS_PER_CATEGORY]
|
||||
for p_rm in prefs_to_remove:
|
||||
for ext in [".xml", ".jpg", ".log", ".meta.json"]:
|
||||
fp = os.path.join(DUMP_DIR, p_rm + ext)
|
||||
if os.path.exists(fp):
|
||||
os.remove(fp)
|
||||
|
||||
# Also clean orphaned files that don't match any known prefix pattern
|
||||
for f in all_files:
|
||||
if "__" not in f:
|
||||
fp = os.path.join(DUMP_DIR, f)
|
||||
if os.path.isfile(fp):
|
||||
os.remove(fp)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"[Diagnostic] Error during dump rotation: {e}")
|
||||
|
||||
@@ -7,12 +7,50 @@ from GramAddict.core.session_state import SessionState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Hard cap: maximum DM replies per inbox visit to prevent spam.
|
||||
MAX_REPLIES_PER_INBOX_VISIT = 3
|
||||
|
||||
# Sentinel values that indicate missing message context.
|
||||
_EMPTY_CONTEXT_SENTINELS = frozenset({"no previous context", "", "none", "n/a"})
|
||||
|
||||
|
||||
# Structural resource-IDs that indicate a real "Send" button.
|
||||
def _is_send_button(node: dict) -> bool:
|
||||
"""Semantic verification: returns True if the node is identified as a Send button."""
|
||||
desc = (node.get("description") or node.get("desc", "")).lower()
|
||||
text = (node.get("text") or "").lower()
|
||||
rid = (node.get("id") or node.get("resource_id", "")).lower()
|
||||
|
||||
# Accept if semantic markers indicate sending
|
||||
if any(m in rid for m in ["send", "composer_button"]):
|
||||
return True
|
||||
if any(m in desc for m in ["send", "absenden"]):
|
||||
return True
|
||||
if text == "send" or text == "absenden":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_state, current_target, cognitive_stack):
|
||||
"""
|
||||
Executes the autonomous Direct Messaging logic in the Zero-Latency architecture.
|
||||
Assumes the bot is already at the "MessageInbox" UI state.
|
||||
|
||||
Safety guarantees:
|
||||
- Refuses to execute if dm_reply plugin is disabled in config.
|
||||
- Skips threads with no extractable text context.
|
||||
- Structurally verifies the Send button before logging success.
|
||||
- Hard-caps replies per inbox visit to MAX_REPLIES_PER_INBOX_VISIT.
|
||||
"""
|
||||
# ── Kill-Switch: Respect dm_reply.enabled config ──
|
||||
dm_plugin_config = configs.get_plugin_config("dm_reply")
|
||||
if not dm_plugin_config.get("enabled", False):
|
||||
logger.warning(
|
||||
"🛑 [DM Engine] dm_reply plugin is DISABLED in config. Refusing to process inbox.",
|
||||
extra={"color": f"{Fore.RED}"},
|
||||
)
|
||||
return "BOREDOM_CHANGE_FEED"
|
||||
|
||||
logger.info(
|
||||
f"🧠 [DM Engine] Initiating inbox processing in {current_target}...",
|
||||
extra={"color": f"{Style.BRIGHT}{Fore.CYAN}"},
|
||||
@@ -20,7 +58,6 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
|
||||
telepathic = cognitive_stack.get("telepathic")
|
||||
dopamine = cognitive_stack.get("dopamine")
|
||||
crm = cognitive_stack.get("crm")
|
||||
|
||||
from GramAddict.core.bot_flow import _humanized_click, sleep
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
@@ -31,6 +68,7 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
session_state.totalMessages = 0
|
||||
|
||||
failed_attempts = 0
|
||||
replies_this_visit = 0
|
||||
|
||||
while not dopamine.is_app_session_over():
|
||||
# Limits check
|
||||
@@ -44,6 +82,33 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
try:
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
# --- Zero Trust Structural Guard ---
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
identity_engine = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
identity_engine.device = device
|
||||
screen_info = identity_engine.identify(xml_dump)
|
||||
|
||||
screen_type = screen_info["screen_type"]
|
||||
is_inbox = screen_type == ScreenType.DM_INBOX
|
||||
is_thread = screen_type == ScreenType.DM_THREAD
|
||||
|
||||
if is_thread:
|
||||
logger.warning("⚠️ [Structural Guard] DM Engine trapped in an open thread. Escaping...")
|
||||
device.press("back")
|
||||
from GramAddict.core.bot_flow import sleep
|
||||
|
||||
sleep(1.5)
|
||||
continue
|
||||
|
||||
if not is_inbox:
|
||||
# We have drifted somewhere entirely alien (like Privacy Settings)
|
||||
logger.error(
|
||||
f"🛑 [Structural Guard] Alien context detected ({screen_type}). Not in Inbox. Triggering CONTEXT_LOST."
|
||||
)
|
||||
return "CONTEXT_LOST"
|
||||
# -----------------------------------
|
||||
|
||||
# Step 1: Find unread conversation threads
|
||||
unread_threads = telepathic._extract_semantic_nodes(
|
||||
xml_dump, "find unread message threads or unread badges", threshold=0.7
|
||||
@@ -67,60 +132,101 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
|
||||
logger.debug(f"Last received message context: {context_text}")
|
||||
|
||||
# Verify we aren't at limits before sending
|
||||
if not getattr(configs.args, "disable_ai_messaging", False):
|
||||
# Configure models
|
||||
model = getattr(configs.args, "ai_condenser_model", "llama3.2:1b")
|
||||
url = getattr(configs.args, "ai_condenser_url", "http://localhost:11434/api/generate")
|
||||
|
||||
# Generate response
|
||||
prompt = f"You are replying to a direct message on Instagram. The last message you received was: '{context_text}'. Keep it short, casual, and friendly. Do not use hashtags."
|
||||
|
||||
response_dict = query_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
format_json=False,
|
||||
timeout=120,
|
||||
max_tokens=100,
|
||||
temperature=0.7,
|
||||
# ── Context Guard: Skip threads with no extractable message ──
|
||||
if context_text.strip().lower() in _EMPTY_CONTEXT_SENTINELS:
|
||||
logger.warning(
|
||||
"⏭️ [DM Engine] Thread has no extractable message context (story reply / media-only). Skipping."
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
continue
|
||||
|
||||
if response_dict and "response" in response_dict:
|
||||
response_text = response_dict["response"].strip()
|
||||
# Find the input field
|
||||
input_nodes = telepathic._extract_semantic_nodes(
|
||||
thread_xml, "find the message input text field", threshold=0.7
|
||||
# Verify we aren't at limits before sending
|
||||
# ── Iteration Cap: Prevent DM spam ──
|
||||
if replies_this_visit >= MAX_REPLIES_PER_INBOX_VISIT:
|
||||
logger.info(
|
||||
f"🛑 [DM Engine] Reached max replies per inbox visit ({MAX_REPLIES_PER_INBOX_VISIT}). Exiting."
|
||||
)
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
return "BOREDOM_CHANGE_FEED"
|
||||
|
||||
# Configure models
|
||||
model = getattr(configs.args, "ai_condenser_model", "llama3.2:1b")
|
||||
url = getattr(configs.args, "ai_condenser_url", "http://localhost:11434/api/generate")
|
||||
|
||||
# Generate response
|
||||
prompt = f"You are replying to a direct message on Instagram. The last message you received was: '{context_text}'. Keep it short, casual, and friendly. Do not use hashtags."
|
||||
|
||||
logger.info(">>> [DM Engine] ABOUT TO CALL LLM")
|
||||
response_dict = query_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
format_json=False,
|
||||
timeout=120,
|
||||
max_tokens=100,
|
||||
temperature=0.7,
|
||||
)
|
||||
logger.info(f">>> [DM Engine] LLM RETURNED: {response_dict}")
|
||||
|
||||
if response_dict and "response" in response_dict:
|
||||
response_text = response_dict["response"].strip()
|
||||
# Find the input field
|
||||
input_nodes = telepathic._extract_semantic_nodes(
|
||||
thread_xml, "find the message input text field", threshold=0.7
|
||||
)
|
||||
if input_nodes and not input_nodes[0].get("skip"):
|
||||
in_node = input_nodes[0]
|
||||
_humanized_click(device, in_node["x"], in_node["y"])
|
||||
sleep(1.0)
|
||||
|
||||
# Type the message
|
||||
ghost_type(device, response_text, speed="fast")
|
||||
sleep(1.0)
|
||||
|
||||
# Find Send button
|
||||
send_xml = device.dump_hierarchy()
|
||||
send_nodes = telepathic._extract_semantic_nodes(
|
||||
send_xml, "find the send message button", threshold=0.8
|
||||
)
|
||||
if input_nodes and not input_nodes[0].get("skip"):
|
||||
in_node = input_nodes[0]
|
||||
_humanized_click(device, in_node["x"], in_node["y"])
|
||||
sleep(1.0)
|
||||
|
||||
# Type the message
|
||||
ghost_type(device, response_text, speed="fast")
|
||||
sleep(1.0)
|
||||
if send_nodes and not send_nodes[0].get("skip"):
|
||||
s_node = send_nodes[0]
|
||||
|
||||
# Find Send button
|
||||
send_xml = device.dump_hierarchy()
|
||||
send_nodes = telepathic._extract_semantic_nodes(
|
||||
send_xml, "find the send message button", threshold=0.8
|
||||
)
|
||||
|
||||
if send_nodes and not send_nodes[0].get("skip"):
|
||||
s_node = send_nodes[0]
|
||||
# ── Send Button Structural Verification ──
|
||||
if not _is_send_button(s_node):
|
||||
s_rid = s_node.get("original_attribs", {}).get("resource-id", "unknown")
|
||||
logger.warning(
|
||||
f"⚠️ [DM Engine] Refused to click non-Send element: {s_rid}. Aborting reply."
|
||||
)
|
||||
else:
|
||||
_humanized_click(device, s_node["x"], s_node["y"])
|
||||
logger.info(
|
||||
"✅ [DM Engine] Successfully sent a generated reply.", extra={"color": Fore.GREEN}
|
||||
"✅ [DM Engine] Successfully sent a generated reply.",
|
||||
extra={"color": Fore.GREEN},
|
||||
)
|
||||
|
||||
session_state.totalMessages += 1
|
||||
if crm:
|
||||
crm.log_sent_dm("unknown_target", response_text, "", [])
|
||||
replies_this_visit += 1
|
||||
dm_memory = cognitive_stack.get("dm_memory")
|
||||
if dm_memory:
|
||||
dm_memory.log_sent_dm("unknown_target", response_text, "", [])
|
||||
|
||||
# Return back to inbox
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
sleep(1.5)
|
||||
|
||||
# If keyboard was open, the first back only closed it. Check if still in thread.
|
||||
check_xml = device.dump_hierarchy()
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
dopamine.boredom += random.uniform(5.0, 15.0)
|
||||
failed_attempts = 0
|
||||
@@ -136,6 +242,19 @@ def _run_zero_latency_dm_loop(device, zero_engine, nav_graph, configs, session_s
|
||||
except Exception as e:
|
||||
logger.error(f"⚠️ [Anomaly Handler] Exception in DM Loop: {e}")
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
check_xml = device.dump_hierarchy()
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
check_identity = ScreenIdentity(getattr(configs.args, "username", ""))
|
||||
check_identity.device = device
|
||||
check_screen = check_identity.identify(check_xml)
|
||||
|
||||
if check_screen["screen_type"] == ScreenType.DM_THREAD:
|
||||
device.press("back")
|
||||
sleep(1.0)
|
||||
|
||||
failed_attempts += 1
|
||||
if failed_attempts > 2:
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
@@ -93,6 +93,18 @@ class DopamineEngine:
|
||||
)
|
||||
|
||||
def is_app_session_over(self):
|
||||
# Global Hard Kill check
|
||||
if getattr(self, "global_max_runtime_minutes", None):
|
||||
if hasattr(self, "global_start_time"):
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
if datetime.now() - self.global_start_time > timedelta(minutes=self.global_max_runtime_minutes):
|
||||
logger.info(
|
||||
f"🛑 [Timeout] Maximum runtime of {self.global_max_runtime_minutes} minutes reached (checked by DopamineEngine). Force-stopping session.",
|
||||
extra={"color": f"{Fore.RED}"},
|
||||
)
|
||||
return True
|
||||
|
||||
# True if we have scrolled too long or hit absolute burnout
|
||||
return (time.time() - self.session_start) > self.session_limit_seconds or self.boredom >= 100.0
|
||||
|
||||
|
||||
276
GramAddict/core/goal_decomposer.py
Normal file
276
GramAddict/core/goal_decomposer.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""
|
||||
GoalDecomposer — Mission-Driven Task Planning
|
||||
|
||||
Translates the bot's `mission` config + `plugins` capabilities into
|
||||
concrete, weighted Task objects. Pure logic — no LLM, no device,
|
||||
no network, no side effects.
|
||||
|
||||
This is the bridge between:
|
||||
- "What does the user WANT?" (mission.strategy)
|
||||
- "What CAN the bot DO?" (enabled plugins + actions)
|
||||
- "What SHOULD it do NOW?" (weighted Task selection)
|
||||
|
||||
Tesla analogy: FSD doesn't have a "goal: drive safely" config.
|
||||
It derives behavior from destination + road rules + sensor capabilities.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Strategy Weight Tables ──
|
||||
# Each strategy defines relative weights for screen targets.
|
||||
# Higher weight = more likely to be selected by GrowthBrain.
|
||||
STRATEGY_WEIGHTS: Dict[str, Dict[str, float]] = {
|
||||
"aggressive_growth": {
|
||||
"HomeFeed": 0.15,
|
||||
"ExploreFeed": 0.45,
|
||||
"ReelsFeed": 0.15,
|
||||
"StoriesFeed": 0.10,
|
||||
"MessageInbox": 0.10,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"community_builder": {
|
||||
"HomeFeed": 0.40,
|
||||
"ExploreFeed": 0.10,
|
||||
"ReelsFeed": 0.05,
|
||||
"StoriesFeed": 0.25,
|
||||
"MessageInbox": 0.15,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"passive_learning": {
|
||||
"HomeFeed": 0.20,
|
||||
"ExploreFeed": 0.50,
|
||||
"ReelsFeed": 0.20,
|
||||
"StoriesFeed": 0.05,
|
||||
"MessageInbox": 0.00,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
"stealth_lurker": {
|
||||
"HomeFeed": 0.35,
|
||||
"ExploreFeed": 0.25,
|
||||
"ReelsFeed": 0.15,
|
||||
"StoriesFeed": 0.15,
|
||||
"MessageInbox": 0.05,
|
||||
"FollowingList": 0.05,
|
||||
},
|
||||
}
|
||||
|
||||
# ── Plugin → Screen Mapping ──
|
||||
# Which plugins enable which screen targets.
|
||||
# A screen is only viable if at least one enabling plugin is active.
|
||||
# Some plugins work on MULTIPLE screens (likes work on home, explore, reels).
|
||||
PLUGIN_SCREENS_MAP: Dict[str, set] = {
|
||||
"likes": {"HomeFeed", "ExploreFeed", "ReelsFeed"},
|
||||
"comment": {"HomeFeed", "ExploreFeed"},
|
||||
"follow": {"HomeFeed", "ExploreFeed"},
|
||||
"repost": {"HomeFeed", "ExploreFeed"},
|
||||
"profile_visit": {"HomeFeed", "ExploreFeed"},
|
||||
"grid_like": {"HomeFeed"},
|
||||
"carousel_browsing": {"HomeFeed"},
|
||||
"rabbit_hole": {"HomeFeed", "ExploreFeed"},
|
||||
"story_view": {"StoriesFeed"},
|
||||
"dm_reply": {"MessageInbox"},
|
||||
}
|
||||
|
||||
# ── Action → Screen Mapping ──
|
||||
# The `actions:` config section maps directly to screens.
|
||||
ACTION_SCREEN_MAP: Dict[str, str] = {
|
||||
"feed": "HomeFeed",
|
||||
"explore": "ExploreFeed",
|
||||
"reels": "ReelsFeed",
|
||||
}
|
||||
|
||||
# ── Screen → Verb Mapping ──
|
||||
SCREEN_VERB_MAP: Dict[str, str] = {
|
||||
"HomeFeed": "browse_feed",
|
||||
"ExploreFeed": "browse_explore",
|
||||
"ReelsFeed": "browse_reels",
|
||||
"StoriesFeed": "view_stories",
|
||||
"MessageInbox": "check_messages",
|
||||
"FollowingList": "manage_following",
|
||||
}
|
||||
|
||||
# ── Screen → Human Intent ──
|
||||
SCREEN_INTENT_MAP: Dict[str, str] = {
|
||||
"HomeFeed": "Interact with posts in the home feed",
|
||||
"ExploreFeed": "Discover and engage with new content",
|
||||
"ReelsFeed": "Browse and interact with reels",
|
||||
"StoriesFeed": "View and react to stories",
|
||||
"MessageInbox": "Reply to unread direct messages",
|
||||
"FollowingList": "Review and manage following list",
|
||||
}
|
||||
|
||||
DEFAULT_BUDGET = 5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Task:
|
||||
"""A concrete, executable unit of work for the bot.
|
||||
|
||||
Unlike abstract goals ("nurture community"), a Task has:
|
||||
- A specific screen to navigate to
|
||||
- A measurable budget (how many posts/items to process)
|
||||
- A weight for probabilistic selection
|
||||
- A human-readable intent for logging
|
||||
"""
|
||||
|
||||
verb: str
|
||||
target_screen: str
|
||||
intent: str
|
||||
budget_posts: int
|
||||
weight: float
|
||||
|
||||
|
||||
class GoalDecomposer:
|
||||
"""Translates mission + plugins → weighted Task list.
|
||||
|
||||
Pure logic, zero side effects. Call generate_tasks() to get
|
||||
the bot's action menu for the current session.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
plugins: Dict[str, dict],
|
||||
actions: Dict[str, str],
|
||||
mission: Dict[str, str],
|
||||
):
|
||||
self._plugins = plugins
|
||||
self._actions = actions
|
||||
self._strategy = mission.get("strategy", "aggressive_growth")
|
||||
|
||||
def generate_tasks(self) -> List[Task]:
|
||||
"""Generate weighted tasks from config.
|
||||
|
||||
Returns an empty list if no plugins are enabled —
|
||||
the bot literally has nothing to do.
|
||||
"""
|
||||
viable_screens = self._discover_viable_screens()
|
||||
if not viable_screens:
|
||||
return []
|
||||
|
||||
strategy_weights = STRATEGY_WEIGHTS.get(self._strategy, STRATEGY_WEIGHTS["aggressive_growth"])
|
||||
|
||||
tasks = []
|
||||
for screen in viable_screens:
|
||||
weight = strategy_weights.get(screen, 0.1)
|
||||
if weight <= 0:
|
||||
continue
|
||||
|
||||
budget = self._budget_for_screen(screen)
|
||||
verb = SCREEN_VERB_MAP.get(screen, "browse")
|
||||
intent = SCREEN_INTENT_MAP.get(screen, f"Interact on {screen}")
|
||||
|
||||
tasks.append(
|
||||
Task(
|
||||
verb=verb,
|
||||
target_screen=screen,
|
||||
intent=intent,
|
||||
budget_posts=budget,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
|
||||
return tasks
|
||||
|
||||
def _discover_viable_screens(self) -> set:
|
||||
"""Determine which screens the bot can meaningfully interact on.
|
||||
|
||||
A screen is viable if it has BOTH:
|
||||
1. A route (action config or plugin-implied), AND
|
||||
2. At least one active plugin that can DO something there.
|
||||
|
||||
Without an active plugin, navigating to a screen is pointless —
|
||||
the bot would just scroll with nothing to interact on.
|
||||
"""
|
||||
# 1. Collect screens with active plugins
|
||||
plugin_screens: set = set()
|
||||
for plugin_name, screens in PLUGIN_SCREENS_MAP.items():
|
||||
plugin_cfg = self._plugins.get(plugin_name, {})
|
||||
if not plugin_cfg:
|
||||
continue
|
||||
if not self._is_plugin_active(plugin_cfg):
|
||||
continue
|
||||
plugin_screens.update(screens)
|
||||
|
||||
# 2. Screens from actions are only viable if plugins exist for them
|
||||
action_screens: set = set()
|
||||
for action_key, screen in ACTION_SCREEN_MAP.items():
|
||||
if action_key in self._actions and self._actions[action_key]:
|
||||
action_screens.add(screen)
|
||||
|
||||
# 3. A screen must have plugin coverage to be viable
|
||||
# Action-enabled screens need at least one active plugin
|
||||
viable = action_screens & plugin_screens
|
||||
|
||||
# 4. Plugin-only screens (story_view, dm_reply) are viable
|
||||
# even without an explicit action config
|
||||
viable |= plugin_screens
|
||||
|
||||
return viable
|
||||
|
||||
def _is_plugin_active(self, plugin_cfg: dict) -> bool:
|
||||
"""Check if a plugin config represents an active plugin.
|
||||
|
||||
A plugin is active if:
|
||||
- It has `enabled: true` (explicit), OR
|
||||
- It has `percentage` > 0 (implicit enable), OR
|
||||
- It has any config keys and `enabled` is not explicitly False
|
||||
"""
|
||||
# Explicit disable
|
||||
if plugin_cfg.get("enabled") is False:
|
||||
return False
|
||||
|
||||
# Explicit enable
|
||||
if plugin_cfg.get("enabled") is True:
|
||||
return True
|
||||
|
||||
# Percentage-based: 0% means disabled
|
||||
pct = plugin_cfg.get("percentage")
|
||||
if pct is not None:
|
||||
try:
|
||||
return float(pct) > 0
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
# Has config keys but no explicit enabled/percentage = active
|
||||
return bool(plugin_cfg)
|
||||
|
||||
def _budget_for_screen(self, screen: str) -> int:
|
||||
"""Determine the post budget for a screen.
|
||||
|
||||
Reads from actions config (e.g. feed: "5-10") and parses
|
||||
the range string into a random integer within bounds.
|
||||
"""
|
||||
# Map screen back to action key
|
||||
reverse_map = {v: k for k, v in ACTION_SCREEN_MAP.items()}
|
||||
action_key = reverse_map.get(screen)
|
||||
|
||||
if action_key and action_key in self._actions:
|
||||
return _parse_range(self._actions[action_key])
|
||||
|
||||
# Special screens get fixed budgets from plugin config
|
||||
if screen == "StoriesFeed":
|
||||
story_cfg = self._plugins.get("story_view", {})
|
||||
count_str = story_cfg.get("count", "1-3")
|
||||
return _parse_range(str(count_str))
|
||||
|
||||
if screen == "MessageInbox":
|
||||
return DEFAULT_BUDGET
|
||||
|
||||
return DEFAULT_BUDGET
|
||||
|
||||
|
||||
def _parse_range(range_str: str) -> int:
|
||||
"""Parse a range string like '5-10' into a random int within bounds."""
|
||||
try:
|
||||
if "-" in str(range_str):
|
||||
parts = str(range_str).split("-")
|
||||
low, high = int(parts[0]), int(parts[1])
|
||||
return random.randint(low, high)
|
||||
return int(range_str)
|
||||
except (ValueError, IndexError):
|
||||
return DEFAULT_BUDGET
|
||||
@@ -17,15 +17,13 @@ import logging
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from GramAddict.core.utils import random_sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
from GramAddict.core.navigation.knowledge import NavigationKnowledge
|
||||
from GramAddict.core.navigation.path_memory import PathMemory
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
from GramAddict.core.utils import random_sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Re-export for backward compatibility (optional but helps minimize import breakage)
|
||||
__all__ = ["GoalExecutor", "ScreenIdentity", "ScreenType", "PathMemory", "NavigationKnowledge", "GoalPlanner"]
|
||||
@@ -45,6 +43,8 @@ class GoalExecutor:
|
||||
"""
|
||||
|
||||
_instance = None
|
||||
global_start_time = None
|
||||
global_max_runtime_minutes = None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, device=None, bot_username=""):
|
||||
@@ -63,6 +63,7 @@ class GoalExecutor:
|
||||
self.device = device
|
||||
self.username = bot_username
|
||||
self.screen_id = ScreenIdentity(bot_username)
|
||||
self.screen_id.device = device
|
||||
self.planner = GoalPlanner(bot_username)
|
||||
self.path_memory = PathMemory(bot_username)
|
||||
self.max_steps = 15 # Safety: never execute more than 15 steps
|
||||
@@ -119,10 +120,25 @@ class GoalExecutor:
|
||||
consecutive_back_presses = 0
|
||||
MAX_CONSECUTIVE_BACK = 3
|
||||
explored_nav_actions = set()
|
||||
visited_screens = set()
|
||||
for step_num in range(max_steps):
|
||||
# ── Global Hard Kill Check ──
|
||||
max_rt = GoalExecutor.global_max_runtime_minutes
|
||||
start_time = GoalExecutor.global_start_time
|
||||
if max_rt and start_time:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
if datetime.now() - start_time > timedelta(minutes=max_rt):
|
||||
logger.error(
|
||||
f"🛑 [Timeout] Maximum runtime of {max_rt} minutes reached during GOAP execution. Hard stopping planner.",
|
||||
extra={"color": "\\033[31m"},
|
||||
)
|
||||
return False
|
||||
|
||||
# PERCEIVE
|
||||
screen = self.perceive()
|
||||
screen_type = screen["screen_type"]
|
||||
visited_screens.add(screen_type)
|
||||
|
||||
if last_screen_type and screen_type != last_screen_type:
|
||||
logger.debug(
|
||||
@@ -136,7 +152,7 @@ class GoalExecutor:
|
||||
original_available = screen.get("available_actions", []).copy()
|
||||
masked_available = []
|
||||
for act in original_available:
|
||||
fail_count = self.action_failures.get(act, 0)
|
||||
fail_count = self.action_failures.get((screen_type, act), 0)
|
||||
if fail_count >= MAX_RETRIES:
|
||||
logger.warning(
|
||||
f"🚫 [GOAP] Masking action '{act}' due to {fail_count} consecutive failures to prevent loops."
|
||||
@@ -146,7 +162,7 @@ class GoalExecutor:
|
||||
screen["available_actions"] = masked_available
|
||||
|
||||
logger.debug(
|
||||
f"📍 [GOAP Step {step_num + 1}] On: {screen_type.value} | "
|
||||
f"📍 [GOAP Step {step_num + 1}] Goal: '{goal}' | On: {screen_type.value} | "
|
||||
f"Available: {screen.get('available_actions', [])[:5]}"
|
||||
)
|
||||
|
||||
@@ -158,13 +174,23 @@ class GoalExecutor:
|
||||
# SAE Feedback Loop!
|
||||
# If we hit this, the LAST action caused an obstacle! Mask it!
|
||||
if last_action and last_screen_type:
|
||||
self.action_failures[last_action] = (
|
||||
self.action_failures.get(last_action, 0) + MAX_RETRIES
|
||||
) # Instantly mask it
|
||||
self.planner.knowledge.learn_trap(last_screen_type, last_action, f"caused_obstacle_{obstacle_name}")
|
||||
logger.warning(
|
||||
f"🛡️ [SAE Feedback] Action '{last_action}' caused an obstacle. Masking aggressively and learned trap."
|
||||
)
|
||||
self.action_failures[(last_screen_type, last_action)] = (
|
||||
self.action_failures.get((last_screen_type, last_action), 0) + MAX_RETRIES
|
||||
) # Instantly mask it for this session
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
if ScreenTopology.is_structural_action(last_screen_type, last_action):
|
||||
logger.warning(
|
||||
f"🛡️ [SAE Feedback] Structural action '{last_action}' caused an obstacle. "
|
||||
f"Masking for this session. (Never burned permanently)"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"🛡️ [SAE Feedback] Content action '{last_action}' caused an obstacle. "
|
||||
f"Masking for this session to break loop, but preventing permanent Qdrant poisoning."
|
||||
)
|
||||
# We specifically DO NOT call self.planner.knowledge.learn_trap here anymore!
|
||||
# Burning dynamic actions like "tap follow button" permanently destroys the bot's capabilities across sessions.
|
||||
|
||||
if not self._get_sae().ensure_clear_screen():
|
||||
if screen_type == ScreenType.FOREIGN_APP:
|
||||
@@ -173,7 +199,13 @@ class GoalExecutor:
|
||||
continue
|
||||
|
||||
# PLAN
|
||||
action = self.planner.plan_next_step(goal, screen, explored_nav_actions=explored_nav_actions)
|
||||
action = self.planner.plan_next_step(
|
||||
goal,
|
||||
screen,
|
||||
explored_nav_actions=explored_nav_actions,
|
||||
action_failures=self.action_failures,
|
||||
visited_screens=visited_screens,
|
||||
)
|
||||
|
||||
if action is None:
|
||||
# Goal achieved!
|
||||
@@ -193,51 +225,80 @@ class GoalExecutor:
|
||||
|
||||
if success:
|
||||
steps_taken.append({"action": action})
|
||||
|
||||
if action == "force start instagram":
|
||||
logger.info("🔄 [GOAP State] App restarted. Purging memory/traps to attempt fresh routing.")
|
||||
self.action_failures.clear()
|
||||
explored_nav_actions.clear()
|
||||
visited_screens.clear()
|
||||
consecutive_back_presses = 0
|
||||
continue
|
||||
|
||||
# Check if it was a navigation action (vs a goal action). If we are not on the required screen,
|
||||
# any action taken is essentially a navigation attempt.
|
||||
explored_nav_actions.add(action)
|
||||
# Reset failures for this action since it eventually succeeded
|
||||
self.action_failures[action] = 0
|
||||
self.action_failures[(screen_type, action)] = 0
|
||||
|
||||
# ── Back-Press Circuit Breaker ──
|
||||
if "scroll" in action.lower():
|
||||
logger.debug(
|
||||
"📍 [GOAP State] Scrolled successfully. Clearing explored actions to allow retrying off-screen elements."
|
||||
)
|
||||
explored_nav_actions.clear()
|
||||
# Keep action_failures for synthetic intents, but clear them for structural actions
|
||||
# so that the HD Map can retry route actions that might now be visible!
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
keys_to_clear = [
|
||||
k
|
||||
for k in self.action_failures.keys()
|
||||
if k[0] == screen_type and ScreenTopology.is_structural_action(screen_type, k[1])
|
||||
]
|
||||
for k in keys_to_clear:
|
||||
del self.action_failures[k]
|
||||
|
||||
# ── Back-Press Circuit Breaker → Escalation ──
|
||||
if action == "press back":
|
||||
consecutive_back_presses += 1
|
||||
if consecutive_back_presses >= MAX_CONSECUTIVE_BACK:
|
||||
logger.error(
|
||||
logger.warning(
|
||||
f"🛑 [GOAP] Back-pressed {MAX_CONSECUTIVE_BACK} times with no screen transition. "
|
||||
f"Aborting goal '{goal}' to prevent app exit."
|
||||
f"Escalating to force restart."
|
||||
)
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
|
||||
|
||||
# Phase 3 GREEN: Unlearn the trap path
|
||||
# Unlearn the trap path
|
||||
from GramAddict.core.qdrant_memory import NavigationMemoryDB
|
||||
|
||||
# We don't know the exact action that got us here easily without analyzing steps_taken,
|
||||
# but we can grab the first action taken from start_screen in this chain if available.
|
||||
if len(steps_taken) > consecutive_back_presses:
|
||||
last_real_action = steps_taken[-consecutive_back_presses - 1]["action"]
|
||||
logger.debug(
|
||||
f"[GOAP Unlearn] last_real_action={last_real_action}, " f"start_screen={start_screen}"
|
||||
)
|
||||
NavigationMemoryDB().unlearn_transition(start_screen, last_real_action)
|
||||
else:
|
||||
logger.debug(
|
||||
f"[GOAP Unlearn] No real action to unlearn. "
|
||||
f"steps={len(steps_taken)}, back_presses={consecutive_back_presses}"
|
||||
)
|
||||
|
||||
return False
|
||||
# ── ESCALATION: Force restart instead of aborting ──
|
||||
app_id = getattr(self.device, "app_id", "com.instagram.android")
|
||||
self.device.app_start(app_id, use_monkey=True)
|
||||
random_sleep(2.0, 3.5)
|
||||
steps_taken.append({"action": "force start instagram"})
|
||||
|
||||
logger.info("🔄 [GOAP Escalation] App restarted. Purging all failure state for fresh attempt.")
|
||||
self.action_failures.clear()
|
||||
explored_nav_actions.clear()
|
||||
visited_screens.clear()
|
||||
consecutive_back_presses = 0
|
||||
continue
|
||||
else:
|
||||
consecutive_back_presses = 0
|
||||
else:
|
||||
self.action_failures[action] = self.action_failures.get(action, 0) + 1
|
||||
self.action_failures[(screen_type, action)] = self.action_failures.get((screen_type, action), 0) + 1
|
||||
# Track failed actions in explored_nav_actions so the planner
|
||||
# knows NOT to return the same synthetic intent again.
|
||||
# Without this, synthetic intents (not in available_actions)
|
||||
# bypass the masking logic and loop forever.
|
||||
explored_nav_actions.add(action)
|
||||
|
||||
if self.action_failures[action] >= MAX_RETRIES:
|
||||
if self.action_failures[(screen_type, action)] >= MAX_RETRIES:
|
||||
# ── Topology Guard: Never poison structural HD Map actions ──
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
@@ -306,6 +367,25 @@ class GoalExecutor:
|
||||
self._get_sae().ensure_clear_screen(max_attempts=3)
|
||||
return False
|
||||
|
||||
# ── Pre-Click Semantic Match Guard ──
|
||||
# For toggle intents (follow/like/save), verify the selected node
|
||||
# semantically matches the intent BEFORE clicking. This prevents
|
||||
# VLM hallucinations from clicking photo grid items when looking
|
||||
# for follow buttons.
|
||||
from GramAddict.core.perception.action_memory import _intent_matches_node
|
||||
|
||||
node_semantic = (
|
||||
f"text: '{best_node.get('text', '')}', "
|
||||
f"desc: '{best_node.get('description', '')}', "
|
||||
f"id: '{best_node.get('id', '')}'"
|
||||
)
|
||||
if not _intent_matches_node(action, node_semantic):
|
||||
logger.warning(
|
||||
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
|
||||
f"Node: {node_semantic}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Execute click
|
||||
self.device.click(obj=best_node)
|
||||
import random
|
||||
@@ -320,11 +400,22 @@ class GoalExecutor:
|
||||
pre_action_screen_type = pre_action_screen["screen_type"]
|
||||
|
||||
# Determine if this was a navigation or an interaction
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
|
||||
if not is_navigation:
|
||||
is_navigation = ScreenTopology.is_structural_action(pre_action_screen_type, action)
|
||||
action_success = False
|
||||
ui_changed = post_xml != xml_dump
|
||||
|
||||
# ── UI Change Detection with Noise Threshold ──
|
||||
# Raw string diffs of < 50 bytes are noise (timestamps, whitespace, counters).
|
||||
# A real navigation changes the XML by hundreds/thousands of bytes.
|
||||
MIN_UI_CHANGE_BYTES = 50
|
||||
xml_delta = abs(len(post_xml) - len(xml_dump))
|
||||
ui_changed = post_xml != xml_dump and xml_delta >= MIN_UI_CHANGE_BYTES
|
||||
logger.debug(
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, " f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}"
|
||||
f"[GOAP Verify] ui_changed={ui_changed}, "
|
||||
f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}, delta={xml_delta}b"
|
||||
)
|
||||
|
||||
if is_navigation:
|
||||
@@ -380,8 +471,18 @@ class GoalExecutor:
|
||||
action_success = False
|
||||
else:
|
||||
# For interactions (like, follow) or unknown goals, use XML delta + semantic verify
|
||||
if ui_changed:
|
||||
verification = engine.verify_success(action, post_xml)
|
||||
# REGRESSION FIX 2026-05-01: Toggle actions (like/save) produce tiny XML deltas
|
||||
# (e.g. checked="false" → "true" = 1 byte). We must NOT gate interactions on
|
||||
# MIN_UI_CHANGE_BYTES. ANY change at all warrants semantic verification.
|
||||
interaction_xml_changed = post_xml != xml_dump
|
||||
if post_screen_type == ScreenType.FOREIGN_APP:
|
||||
logger.error(
|
||||
f"❌ [GOAP Verify] Interaction '{action}' caused navigation to FOREIGN_APP (e.g. Play Store). Rejecting as catastrophic failure."
|
||||
)
|
||||
action_success = False
|
||||
elif interaction_xml_changed:
|
||||
score = best_node.get("score", 0.0) if best_node else 0.0
|
||||
verification = engine.verify_success(action, post_xml, device=self.device, confidence=score)
|
||||
if verification is True:
|
||||
action_success = True
|
||||
logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.")
|
||||
@@ -412,8 +513,12 @@ class GoalExecutor:
|
||||
return False
|
||||
else:
|
||||
# action_success is None (INCONCLUSIVE)
|
||||
# We decay the memory so it unlearns if it repeatedly fails to produce a definitive success.
|
||||
logger.warning(f"⚠️ [GOAP Execute] Applying AGGRESSIVE PENALTY for inconclusive action '{action}'.")
|
||||
engine.decay_click(action)
|
||||
# Double penalty to burn ambiguous paths faster (outer loop adds +1, so total +2 = instantly hits MAX_RETRIES)
|
||||
self.action_failures[(pre_action_screen_type, action)] = (
|
||||
self.action_failures.get((pre_action_screen_type, action), 0) + 1
|
||||
)
|
||||
return False
|
||||
|
||||
def _execute_recalled_path(self, steps: List[Dict], goal: str) -> bool:
|
||||
|
||||
@@ -94,6 +94,63 @@ class GrowthBrain:
|
||||
logger.info(f"🧠 [GrowthBrain] Strategy '{self.strategy}' dictated Desire: {selected_desire}")
|
||||
return selected_desire
|
||||
|
||||
def get_current_goal(self, dopamine_engine, available_goals: list[str], success_rates: dict = None) -> str:
|
||||
"""
|
||||
Autonomously selects the next strategic goal.
|
||||
If no goals are configured, falls back to legacy desires.
|
||||
Weights goals based on session success rates if provided.
|
||||
|
||||
.. deprecated::
|
||||
Use select_task() instead for concrete, plugin-linked task selection.
|
||||
"""
|
||||
import random
|
||||
|
||||
if not available_goals:
|
||||
# Legacy Desire Mapping (Fallback)
|
||||
return self.get_current_desire(dopamine_engine)
|
||||
|
||||
if dopamine_engine.boredom > 80:
|
||||
return "ShiftContext" # High boredom triggers a context shift
|
||||
|
||||
if not success_rates:
|
||||
return random.choice(available_goals)
|
||||
|
||||
weights = []
|
||||
for goal in available_goals:
|
||||
base_weight = 1.0
|
||||
success_count = success_rates.get(goal, 0)
|
||||
weight = base_weight + float(success_count)
|
||||
weights.append(weight)
|
||||
|
||||
return random.choices(available_goals, weights=weights, k=1)[0]
|
||||
|
||||
def select_task(self, dopamine_engine, available_tasks: list) -> "Optional[Task]":
|
||||
"""Select the next concrete Task using weighted random selection.
|
||||
|
||||
This is the primary interface for the orchestrator. Unlike get_current_goal()
|
||||
which returns abstract strings, this returns a Task object with a specific
|
||||
target_screen, budget, and success metric.
|
||||
|
||||
Returns:
|
||||
Task: The selected task to execute.
|
||||
None: If no tasks available or boredom is too high (ShiftContext signal).
|
||||
"""
|
||||
if not available_tasks:
|
||||
return None
|
||||
|
||||
# High boredom = ShiftContext (take a break, switch feed)
|
||||
if dopamine_engine.boredom > 85.0:
|
||||
logger.info("🧠 [GrowthBrain] Boredom too high for task selection. ShiftContext.")
|
||||
return None
|
||||
|
||||
weights = [task.weight for task in available_tasks]
|
||||
selected = random.choices(available_tasks, weights=weights, k=1)[0]
|
||||
logger.info(
|
||||
f"🧠 [GrowthBrain] Selected task: {selected.verb} → {selected.target_screen} "
|
||||
f"(weight={selected.weight:.2f}, budget={selected.budget_posts})"
|
||||
)
|
||||
return selected
|
||||
|
||||
def get_circadian_pacing(self) -> float:
|
||||
"""
|
||||
Adjusts activity levels based on the current local time
|
||||
|
||||
86
GramAddict/core/interaction.py
Normal file
86
GramAddict/core/interaction.py
Normal file
@@ -0,0 +1,86 @@
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMWriter:
|
||||
"""
|
||||
The Creative Engine — Content Generation for Interactions.
|
||||
|
||||
Generates high-fidelity, persona-aligned comments and messages.
|
||||
Replaces legacy static 'comment_list' with dynamic, contextual resonance.
|
||||
"""
|
||||
|
||||
def __init__(self, username: str, persona_interests: list[str], configs):
|
||||
self.username = username
|
||||
self.persona_interests = persona_interests
|
||||
self.configs = configs
|
||||
self.args = getattr(configs, "args", None)
|
||||
|
||||
def generate_comment(self, post_data: Dict) -> str:
|
||||
"""
|
||||
Generates a human-like comment based on post data and persona interests.
|
||||
"""
|
||||
if not post_data:
|
||||
logger.warning("✍️ [Writer] No post data provided. Using generic fallback.")
|
||||
return "Cool!"
|
||||
|
||||
caption = post_data.get("caption", "")
|
||||
description = post_data.get("description", "")
|
||||
target_username = post_data.get("username", "the user")
|
||||
|
||||
# Build context for the LLM
|
||||
context = f"Post by @{target_username}\n"
|
||||
if caption:
|
||||
context += f"Caption: {caption}\n"
|
||||
if description:
|
||||
context += f"Visual Description: {description}\n"
|
||||
|
||||
interests_str = ", ".join(self.persona_interests) if self.persona_interests else "general interesting things"
|
||||
|
||||
prompt = (
|
||||
f"You are an Instagram user interested in: {interests_str}.\n"
|
||||
f"You want to leave a brief, friendly, and authentic comment on the following post:\n\n"
|
||||
f"{context}\n"
|
||||
f"INSTRUCTIONS:\n"
|
||||
f"1. Keep it under 10 words.\n"
|
||||
f"2. Be casual and human. Avoid overly formal language or sounding like a bot.\n"
|
||||
f"3. Do NOT use more than one emoji.\n"
|
||||
f"4. Do NOT use hashtags.\n"
|
||||
f"5. Focus on something specific in the post if possible.\n"
|
||||
f"6. Reply with ONLY the comment text."
|
||||
)
|
||||
|
||||
model = getattr(self.args, "ai_writer_model", getattr(self.args, "ai_model", "llama3.2:1b"))
|
||||
url = getattr(
|
||||
self.args, "ai_writer_url", getattr(self.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
)
|
||||
|
||||
logger.info(f"✍️ [Writer] Generating comment for @{target_username} using {model}...")
|
||||
|
||||
try:
|
||||
response_dict = query_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
system="You are a friendly Instagram user. You write short, authentic comments.",
|
||||
format_json=False,
|
||||
timeout=60,
|
||||
temperature=0.7, # Add some variety to avoid 'the to the' loops
|
||||
)
|
||||
|
||||
if response_dict and "response" in response_dict:
|
||||
comment = response_dict["response"].strip().strip('"')
|
||||
# Basic cleaning to remove LLM artifacts
|
||||
comment = comment.split("\n")[0] # Take only first line
|
||||
if not comment:
|
||||
return "Nice!"
|
||||
return comment
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"✍️ [Writer] Failed to generate comment: {e}")
|
||||
|
||||
return "Great post! 🔥"
|
||||
@@ -287,6 +287,12 @@ def query_llm(
|
||||
req_data["images"] = images_b64
|
||||
if format_json:
|
||||
req_data["format"] = "json"
|
||||
else:
|
||||
# For free-text calls (Brain action extraction), explicitly disable
|
||||
# thinking mode. Reasoning models like qwen3.5 put EVERYTHING in
|
||||
# the thinking block and return response='', which is useless for
|
||||
# action extraction. think=false forces a direct response.
|
||||
req_data["think"] = False
|
||||
|
||||
# Ollama passes configs inside 'options'
|
||||
if temperature is not None or max_tokens is not None:
|
||||
@@ -344,16 +350,27 @@ def query_llm(
|
||||
return {"response": content}
|
||||
else:
|
||||
# Ollama returns response OR thinking (for reasoning models)
|
||||
content = resp_json.get("response") or resp_json.get("thinking") or ""
|
||||
raw_response = resp_json.get("response", "")
|
||||
raw_thinking = resp_json.get("thinking", "")
|
||||
|
||||
logger.debug(f"DEBUG LLM PAYLOAD: response='{raw_response}', thinking='{raw_thinking}'")
|
||||
|
||||
# CRITICAL: For free-text mode (format_json=False), do NOT substitute
|
||||
# thinking for empty response. The thinking block is REASONING, not
|
||||
# a decision. The Brain parser would extract random actions from it.
|
||||
# For JSON mode (format_json=True), falling back to thinking IS correct
|
||||
# because reasoning models may place structured output in the thinking block.
|
||||
if format_json:
|
||||
content = raw_response or raw_thinking or ""
|
||||
extracted = extract_json(content)
|
||||
if not extracted:
|
||||
# Log more context if JSON extraction fails
|
||||
logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...")
|
||||
raise ValueError("Ollama returned non-JSON content when JSON was expected.")
|
||||
resp_json["response"] = extracted
|
||||
logger.warning(f"Failed to extract JSON from content: {content[:100]}")
|
||||
else:
|
||||
content = extracted
|
||||
else:
|
||||
content = raw_response
|
||||
|
||||
return resp_json
|
||||
return {"response": content}
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?")
|
||||
except Exception as e:
|
||||
|
||||
87
GramAddict/core/navigation/brain.py
Normal file
87
GramAddict/core/navigation/brain.py
Normal file
@@ -0,0 +1,87 @@
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ask_brain_for_action(
|
||||
goal: str, screen_type: str, available_actions: List[str], explored_actions: set, context: dict = None
|
||||
) -> Optional[str]:
|
||||
"""Asks the VLM to decide the best available action to reach the goal, considering failures."""
|
||||
if not available_actions:
|
||||
return None
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
prompt = (
|
||||
f"You are an autonomous Instagram agent. Your ultimate goal is: '{goal}'.\n"
|
||||
f"You are currently on the screen: {screen_type}.\n"
|
||||
f"These actions are available to you right now: {available_actions}\n"
|
||||
)
|
||||
if explored_actions:
|
||||
prompt += f"You recently tried these actions but they failed or didn't help: {list(explored_actions)}\n"
|
||||
if context:
|
||||
prompt += f"Context: {context}\n"
|
||||
|
||||
prompt += (
|
||||
"INSTRUCTIONS:\n"
|
||||
"1. Reason about where you are. Consider the screen type and what actions make sense on that screen.\n"
|
||||
"2. If the goal requires navigating away from the current screen, choose the action that moves you closest to the goal.\n"
|
||||
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). If your target is likely on this screen but not currently visible, you MUST choose 'scroll down'.\n"
|
||||
"4. 'press back' exits the current screen and returns to the previous one. Use it when you are on a screen that doesn't lead to your goal.\n"
|
||||
"5. DO NOT hallucinate actions. Reply ONLY with the exact string from the available actions list.\n"
|
||||
"6. Reply with ONLY the action string, nothing else."
|
||||
)
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt="Choose the next best action.",
|
||||
system=prompt,
|
||||
format_json=False,
|
||||
max_tokens=250,
|
||||
)
|
||||
if response:
|
||||
result = response if isinstance(response, str) else response.get("response", "")
|
||||
result = result.strip().strip("'\"")
|
||||
|
||||
# 1. Exact match check (ideal case)
|
||||
for act in available_actions:
|
||||
if act.lower() == result.lower():
|
||||
return act
|
||||
|
||||
# 2. Strict line-by-line check (often the model outputs the action on the last line)
|
||||
for line in reversed(result.splitlines()):
|
||||
line = line.strip().strip("'\"")
|
||||
for act in available_actions:
|
||||
if act.lower() == line.lower():
|
||||
return act
|
||||
|
||||
# 3. Fuzzy match (find the LAST mentioned action in the text, assuming it's the conclusion)
|
||||
best_act = None
|
||||
best_idx = -1
|
||||
for act in available_actions:
|
||||
idx = result.lower().rfind(act.lower())
|
||||
if idx > best_idx:
|
||||
best_idx = idx
|
||||
best_act = act
|
||||
|
||||
if best_act:
|
||||
logger.warning(f"🧠 [Brain] Extracted action '{best_act}' from verbose LLM output.")
|
||||
return best_act
|
||||
|
||||
logger.warning(f"🧠 [Brain] LLM returned an invalid action or no action found: '{result[:100]}...'. Falling back.")
|
||||
except Exception as e:
|
||||
logger.debug(f"🧠 [Brain] Error querying LLM: {e}")
|
||||
|
||||
return None
|
||||
@@ -109,7 +109,7 @@ class PathMemory:
|
||||
try:
|
||||
from qdrant_client import models
|
||||
|
||||
point_id = self._db._get_id(seed)
|
||||
point_id = self._db.generate_uuid(seed)
|
||||
self._db.client.delete(
|
||||
collection_name=self._db.collection_name, points_selector=models.PointIdsList(points=[point_id])
|
||||
)
|
||||
|
||||
@@ -17,7 +17,14 @@ class GoalPlanner:
|
||||
def __init__(self, username: str):
|
||||
self.knowledge = NavigationKnowledge(username)
|
||||
|
||||
def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]:
|
||||
def plan_next_step(
|
||||
self,
|
||||
goal: str,
|
||||
screen: Dict[str, Any],
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""Plans the NEXT single action to take toward the goal."""
|
||||
screen_type = screen["screen_type"]
|
||||
available = screen.get("available_actions", [])
|
||||
@@ -34,7 +41,9 @@ class GoalPlanner:
|
||||
|
||||
# ── 3. Am I on the right screen? If not, navigate there ──
|
||||
selected_tab = screen.get("selected_tab")
|
||||
nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions)
|
||||
nav_action = self._plan_navigation(
|
||||
goal_lower, screen_type, available, selected_tab, explored_nav_actions, action_failures, visited_screens
|
||||
)
|
||||
if nav_action:
|
||||
return nav_action
|
||||
|
||||
@@ -70,6 +79,8 @@ class GoalPlanner:
|
||||
available: List[str],
|
||||
selected_tab: Optional[str] = None,
|
||||
explored_nav_actions: set = None,
|
||||
action_failures: dict = None,
|
||||
visited_screens: set = None,
|
||||
) -> Optional[str]:
|
||||
"""If we're on the wrong screen, figure out how to navigate.
|
||||
|
||||
@@ -89,10 +100,67 @@ class GoalPlanner:
|
||||
logger.debug(f"🛡️ [Aversive Filter] Masking trapped action: '{action}'")
|
||||
available = safe_available
|
||||
|
||||
# ── 1. HD Map Routing (Primary Strategy) ──
|
||||
visited_screens = visited_screens or set()
|
||||
|
||||
# 0b. No-Op Guard & Anti-Loop Guard:
|
||||
# - Strip tab actions that navigate to the CURRENT screen.
|
||||
# - Strip actions that navigate to PREVIOUSLY VISITED screens (except back-tracking).
|
||||
noop_actions = set()
|
||||
for action in available:
|
||||
expected = ScreenTopology.expected_screen_for_action(action, screen_type)
|
||||
if expected == screen_type:
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [No-Op Guard] Stripping '{action}' — leads back to {screen_type.name}")
|
||||
elif expected in visited_screens and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — leads to visited {expected.name}")
|
||||
|
||||
# Also strip actions where the HD Map says they go TO the current screen from OTHER screens
|
||||
for src_screen, transitions in ScreenTopology.TRANSITIONS.items():
|
||||
if src_screen == screen_type:
|
||||
continue # We already handled this screen's own transitions
|
||||
for action, dest in transitions.items():
|
||||
if dest == screen_type and action in available:
|
||||
noop_actions.add(action)
|
||||
logger.debug(
|
||||
f"🛡️ [No-Op Guard] Stripping '{action}' — known to navigate to current {screen_type.name}"
|
||||
)
|
||||
elif dest in visited_screens and action in available and action != "press back":
|
||||
noop_actions.add(action)
|
||||
logger.debug(f"🛡️ [Anti-Loop Guard] Stripping '{action}' — known to navigate to visited {dest.name}")
|
||||
|
||||
available = [a for a in available if a not in noop_actions]
|
||||
|
||||
# Build avoid_actions for HD Map route planning
|
||||
avoid_actions = (explored_nav_actions or set()).copy()
|
||||
if action_failures:
|
||||
for key, count in action_failures.items():
|
||||
if isinstance(key, tuple) and len(key) == 2:
|
||||
scr, act = key
|
||||
if scr == screen_type and count >= 2: # MAX_RETRIES is 2 in goap
|
||||
avoid_actions.add(act)
|
||||
else:
|
||||
if count >= 2:
|
||||
avoid_actions.add(key)
|
||||
|
||||
target_screen = ScreenTopology.goal_to_target_screen(goal)
|
||||
|
||||
# ── 1. HD Map Pre-Check for Dead Ends ──
|
||||
# If the topological map KNOWS the target is unreachable due to action_failures,
|
||||
# we must preempt the Brain from blindly routing into a dead end.
|
||||
if target_screen and target_screen != screen_type:
|
||||
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
|
||||
if route is None and ScreenTopology.find_route(screen_type, target_screen):
|
||||
logger.warning(
|
||||
f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── 2. HD Map Routing (Primary Strategy for Navigation) ──
|
||||
# Ground UI transitions in structural invariants. If the topological map knows the route, use it.
|
||||
target_screen = ScreenTopology.goal_to_target_screen(goal)
|
||||
if target_screen and target_screen != screen_type:
|
||||
route = ScreenTopology.find_route(screen_type, target_screen)
|
||||
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
|
||||
if route:
|
||||
next_action, next_screen = route[0]
|
||||
# Verify action isn't explored/trapped
|
||||
@@ -104,9 +172,20 @@ class GoalPlanner:
|
||||
)
|
||||
return next_action
|
||||
else:
|
||||
logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Falling back.")
|
||||
logger.warning(f"🛡️ [HD Map] Route action '{next_action}' is trapped. Skipping HD Map.")
|
||||
else:
|
||||
logger.debug(f"🛡️ [HD Map] Route action '{next_action}' already explored. Falling back.")
|
||||
logger.debug(
|
||||
f"🛡️ [HD Map] Route action '{next_action}' already explored and failed. Skipping HD Map."
|
||||
)
|
||||
|
||||
# ── 3. Brain-Driven Decision Making (Fallback / Discovery) ──
|
||||
# For non-navigation goals or when the HD Map is incomplete.
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
brain_action = ask_brain_for_action(goal, screen_type.name, available, avoid_actions)
|
||||
if brain_action:
|
||||
logger.info(f"🧠 [Brain] Decided to execute: '{brain_action}' (to achieve: '{goal}')")
|
||||
return brain_action
|
||||
|
||||
# ── 2. Learned Knowledge (Qdrant) ──
|
||||
required_screens = self.knowledge.get_requirements(goal)
|
||||
@@ -131,7 +210,7 @@ class GoalPlanner:
|
||||
# 5. Find the action we need to take (from learned knowledge or HD map)
|
||||
for target_screen in required_screens:
|
||||
# Try HD Map first!
|
||||
route = ScreenTopology.find_route(screen_type, target_screen)
|
||||
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
|
||||
if route:
|
||||
next_action, next_screen = route[0]
|
||||
if next_action not in (explored_nav_actions or set()):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
@@ -6,6 +7,55 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parse_yes_no(response: str) -> Optional[bool]:
|
||||
"""Parses a VLM response to find a definitive YES or NO without substring-matching 'not' or 'now'."""
|
||||
text = response.strip()
|
||||
|
||||
# Try parsing as JSON first
|
||||
if text.startswith("{"):
|
||||
try:
|
||||
data = json.loads(text)
|
||||
for k, v in data.items():
|
||||
if str(k).strip().upper() == "YES" or str(v).strip().upper() == "YES":
|
||||
return True
|
||||
if str(k).strip().upper() == "NO" or str(v).strip().upper() == "NO":
|
||||
return False
|
||||
if str(k).strip().lower() == "success" and isinstance(v, bool):
|
||||
return v
|
||||
|
||||
# If it is valid JSON but we couldn't definitively find YES/NO,
|
||||
# do NOT fall through to text matching
|
||||
return None
|
||||
except Exception:
|
||||
# Prevent JSON parsing fall-throughs
|
||||
return None
|
||||
|
||||
text_lower = text.lower()
|
||||
if text_lower.startswith("yes"):
|
||||
return True
|
||||
if text_lower.startswith("no") and not text_lower.startswith("now") and not text_lower.startswith("not"):
|
||||
return False
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Semantic Match Keywords — SSOT for intent → element validation
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
# Maps toggle-intent keywords to required element markers.
|
||||
# If the intent contains the key, the clicked element MUST
|
||||
# contain at least one of the corresponding markers in its
|
||||
# text, content_desc, or resource_id.
|
||||
# ZERO MAINTENANCE: Only English words and resource_id fragments allowed.
|
||||
# No localized strings — the bot must work on any device language.
|
||||
TOGGLE_INTENT_MARKERS = {
|
||||
"follow": ["follow", "button_follow"],
|
||||
"like": ["like", "heart", "button_like"],
|
||||
"save": ["save", "saved", "bookmark"],
|
||||
}
|
||||
|
||||
|
||||
class ActionMemory:
|
||||
"""
|
||||
Handles the caching, tracking, and negative reinforcement (unlearning) of UI interactions.
|
||||
@@ -36,7 +86,11 @@ class ActionMemory:
|
||||
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
|
||||
|
||||
def confirm_click(self, intent: str = None):
|
||||
"""Positive Reinforcement: Confirms the last click was successful."""
|
||||
"""Positive Reinforcement: Confirms the last click was successful.
|
||||
|
||||
Guard: Refuses to store in Qdrant if the clicked element does not
|
||||
semantically match the intent. Prevents memory poisoning.
|
||||
"""
|
||||
ctx = self._last_click_context
|
||||
if not ctx:
|
||||
return
|
||||
@@ -44,7 +98,19 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
|
||||
# ── Semantic Mismatch Guard ──
|
||||
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' — "
|
||||
f"clicked element does not match intent: {ctx['semantic_string']}"
|
||||
)
|
||||
self._last_click_context = None
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.",
|
||||
extra={"color": "\x1b[32m"},
|
||||
)
|
||||
|
||||
# Store or boost in Qdrant
|
||||
try:
|
||||
@@ -68,7 +134,9 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
logger.warning(f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.")
|
||||
logger.warning(
|
||||
f"❌ [ActionMemory] Click failed for '{ctx['intent']}'. Applying penalty.", extra={"color": "\x1b[31m"}
|
||||
)
|
||||
|
||||
try:
|
||||
self.ui_memory.decay_confidence(ctx["intent"], ctx["xml_context"])
|
||||
@@ -77,20 +145,225 @@ class ActionMemory:
|
||||
|
||||
self._last_click_context = None
|
||||
|
||||
def verify_success(self, intent: str, pre_click_xml: str, post_click_xml: str) -> Optional[bool]:
|
||||
def verify_success(
|
||||
self, intent: str, pre_click_xml: str, post_click_xml: str, device=None, confidence: float = 0.0
|
||||
) -> Optional[bool]:
|
||||
"""
|
||||
Structural verification: Did the UI actually change after the click?
|
||||
Structural and Visual verification: Did the UI actually change after the click?
|
||||
"""
|
||||
# Specific check for explore grid
|
||||
if "first image in explore grid" in intent or "grid item" in intent:
|
||||
if "row_feed_photo_imageview" in post_click_xml or "row_feed_button_like" in post_click_xml:
|
||||
intent_lower = intent.lower()
|
||||
post_xml_lower = post_click_xml.lower()
|
||||
|
||||
# Specific check for opening a post (from explore/profile grid)
|
||||
if "view a post" in intent_lower or "first image" in intent_lower or "grid item" in intent_lower:
|
||||
if (
|
||||
"row_feed_photo_imageview" in post_xml_lower
|
||||
or "row_feed_button_like" in post_xml_lower
|
||||
or "clips_viewer_view_pager" in post_xml_lower
|
||||
):
|
||||
return True
|
||||
if "explore_action_bar" in post_click_xml and "row_feed_button_like" not in post_click_xml:
|
||||
return None # Still on grid, inconclusive
|
||||
if (
|
||||
"explore_action_bar" in post_xml_lower
|
||||
and "row_feed_button_like" not in post_xml_lower
|
||||
and "clips_viewer" not in post_xml_lower
|
||||
):
|
||||
logger.warning(f"⚠️ [ActionMemory] Still on grid after trying to '{intent}'. Verification FAIL.")
|
||||
return False # Still on grid, definitely failed
|
||||
|
||||
if abs(len(pre_click_xml) - len(post_click_xml)) > 50:
|
||||
logger.debug(f"🧠 [ActionMemory] Structural change detected for '{intent}'. Verification PASS.")
|
||||
return True
|
||||
# Specific check for opening a profile
|
||||
if "profile" in intent_lower or "author" in intent_lower or "username" in intent_lower:
|
||||
if "profile_header_container" in post_xml_lower:
|
||||
logger.info("✅ [ActionMemory] Structural check confirmed profile navigation success.")
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Profile header NOT found after trying to '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
|
||||
return False
|
||||
# Specific check for navigating to Home Feed
|
||||
if "home feed" in intent_lower or "home tab" in intent_lower:
|
||||
if "main_feed_action_bar" in post_xml_lower:
|
||||
logger.info("✅ [ActionMemory] Structural check confirmed Home Feed navigation success.")
|
||||
return True
|
||||
|
||||
# Specific check for navigating to Explore Feed
|
||||
if "explore feed" in intent_lower or "explore tab" in intent_lower or "search" in intent_lower:
|
||||
if "explore_action_bar" in post_xml_lower or "action_bar_search_edit_text" in post_xml_lower:
|
||||
logger.info("✅ [ActionMemory] Structural check confirmed Explore Feed navigation success.")
|
||||
return True
|
||||
|
||||
state_toggles = ["like", "save", "follow", "heart"]
|
||||
is_toggle = any(t in intent_lower for t in state_toggles)
|
||||
|
||||
# ── VLM Verification Fallback ──
|
||||
|
||||
# If we are highly confident (e.g. pulled from Qdrant memory), bypass heavy VLM
|
||||
if device and confidence < 0.95:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Confidence ({confidence:.2f}) < 0.95. Handing over verification for '{intent}' to VLM visual analysis..."
|
||||
)
|
||||
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
|
||||
|
||||
evaluator = SemanticEvaluator()
|
||||
|
||||
# Build context of what was actually clicked
|
||||
clicked_context = ""
|
||||
if self._last_click_context:
|
||||
clicked_context = f"The element that was tapped: {self._last_click_context['semantic_string']}. "
|
||||
|
||||
# Ask VLM to be the absolute source of truth
|
||||
prompt = (
|
||||
f"The user just attempted to perform the action: '{intent}'. "
|
||||
f"{clicked_context}"
|
||||
f"Look at the current screen carefully. Was the action successful? "
|
||||
)
|
||||
if is_toggle:
|
||||
prompt += (
|
||||
"If the intent was 'follow', does the button now indicate 'Following' or 'Requested'? "
|
||||
"If it was 'like', is the heart icon clearly active/red? "
|
||||
"If the screen shifted completely to a profile when you just wanted to like/follow from a feed, it FAILED. "
|
||||
"If the tapped element does NOT sound like a like/follow button (e.g. it's a caption, comment field, or post content), it FAILED. "
|
||||
)
|
||||
else:
|
||||
prompt += (
|
||||
f"Does the current screen match the expected outcome of '{intent}'? "
|
||||
f"For example, if the intent was to open a post/photo, are you looking at a post view (not a user profile or story)? "
|
||||
f"If the intent was to open a profile, are you on a profile page? "
|
||||
f"If the intent was to go back, are you on the previous screen? "
|
||||
)
|
||||
prompt += "Answer ONLY with the word YES or NO."
|
||||
|
||||
try:
|
||||
screenshot = device.get_screenshot_b64()
|
||||
if not screenshot:
|
||||
raise ValueError("No screenshot available from device")
|
||||
response = evaluator._query_vlm(prompt, screenshot)
|
||||
|
||||
decision = _parse_yes_no(response) if response else None
|
||||
|
||||
if decision is True:
|
||||
logger.debug(f"🧠 [ActionMemory] VLM visually confirmed success for '{intent}'.")
|
||||
return True
|
||||
elif decision is False:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] VLM visual verification FAILED for '{intent}'. VLM replied: '{response}'"
|
||||
)
|
||||
return False
|
||||
else:
|
||||
# VLM returned ambiguous response (JSON, mixed signals, etc.)
|
||||
# Don't treat as hard failure — fall through to structural delta verification
|
||||
logger.info(
|
||||
f"🧠 [ActionMemory] VLM response for '{intent}' was not YES/NO "
|
||||
f"(got: '{response[:80]}...'). Falling through to structural verification."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to query VLM for visual verification: {e}")
|
||||
# Fallthrough to structural delta if VLM crashes
|
||||
|
||||
# ── Pre-Structural Semantic Gate ──
|
||||
if is_toggle and self._last_click_context:
|
||||
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
|
||||
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
# Fallback to structural delta
|
||||
logger.info(f"DEBUG: len(pre_click_xml)={len(pre_click_xml)} len(post_click_xml)={len(post_click_xml)}")
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
logger.info(f"DEBUG: diff={diff}")
|
||||
|
||||
if is_toggle:
|
||||
if diff > 1000:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Massive structural shift ({diff} chars) for state-toggle '{intent}'. Navigated away by mistake? Verification FAIL."
|
||||
)
|
||||
return False
|
||||
if diff > 0:
|
||||
logger.debug(f"🧠 [ActionMemory] Structural delta detected for toggle '{intent}'. Verification PASS.")
|
||||
return True
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Zero structural shift (diff={diff}) for state-toggle '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
|
||||
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
|
||||
# also produces a large diff but achieves the wrong goal.
|
||||
if diff > 50:
|
||||
# Is it a standard structural transition?
|
||||
from GramAddict.core.screen_topology import ScreenTopology
|
||||
|
||||
# We don't have screen type here, so we just check if it's in the HD Map keys
|
||||
logger.info(f"DEBUG: intent is '{intent}'")
|
||||
logger.info(
|
||||
f"DEBUG: TRANSITIONS keys are: {[list(t.keys()) for t in ScreenTopology.TRANSITIONS.values()]}"
|
||||
)
|
||||
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
|
||||
logger.info(f"DEBUG: is_standard={is_standard}")
|
||||
|
||||
if is_standard:
|
||||
logger.debug(
|
||||
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.info(
|
||||
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
|
||||
)
|
||||
# For abstract intents, we must visually verify if it actually helped!
|
||||
# If device is available, we use VLM. If not, we fail safe.
|
||||
if device:
|
||||
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
|
||||
|
||||
evaluator = SemanticEvaluator()
|
||||
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
|
||||
try:
|
||||
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
|
||||
decision = _parse_yes_no(response) if response else None
|
||||
if decision is True:
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'. Response: '{response}'"
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"VLM visual verification failed: {e}")
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
|
||||
return False
|
||||
|
||||
# If diff <= 50 for non-toggle
|
||||
logger.warning(
|
||||
f"⚠️ [ActionMemory] Insufficient structural change (diff={diff}) for non-toggle '{intent}'. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
"""Checks if the clicked element semantically matches the toggle intent.
|
||||
|
||||
For toggle intents (follow, like, save), the clicked element MUST contain
|
||||
at least one of the required keywords in its text/desc/id. This prevents
|
||||
photo grid items, captions, and other unrelated elements from being
|
||||
falsely confirmed as successful interactions.
|
||||
|
||||
For non-toggle intents, returns True (no restriction).
|
||||
"""
|
||||
intent_lower = intent.lower()
|
||||
semantic_lower = semantic_string.lower()
|
||||
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in semantic_lower for marker in required_markers):
|
||||
return True
|
||||
logger.debug(
|
||||
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
|
||||
f"{required_markers} but element has: {semantic_string}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Non-toggle intents pass through
|
||||
return True
|
||||
|
||||
@@ -46,15 +46,15 @@ def has_carousel_in_view(xml_dump: str) -> bool:
|
||||
return any(ind in xml_dump for ind in CAROUSEL_INDICATORS)
|
||||
|
||||
|
||||
def extract_post_content(context_xml: str) -> dict:
|
||||
def extract_post_content(context_xml: str, device=None) -> dict:
|
||||
"""
|
||||
Extracts meaningful content data from the current feed post's XML.
|
||||
This is the BOT'S EYES — what it actually "sees" about each post.
|
||||
|
||||
Returns:
|
||||
{'username': str, 'description': str, 'caption': str}
|
||||
{'username': str, 'description': str, 'caption': str, 'username_missing': bool}
|
||||
"""
|
||||
result = {"username": "", "description": "", "caption": ""}
|
||||
result = {"username": "", "description": "", "caption": "", "username_missing": False}
|
||||
|
||||
try:
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
@@ -62,16 +62,87 @@ def extract_post_content(context_xml: str) -> dict:
|
||||
telepath = TelepathicEngine.get_instance()
|
||||
|
||||
# 1. Learn/extract post author dynamically
|
||||
author_node = telepath.find_best_node(context_xml, "post author username header", min_confidence=0.75)
|
||||
# 🛡️ Structural Fast-Path: Prioritize deterministic IDs over AI guesses
|
||||
# Try structural ID fast-path first (100% deterministic)
|
||||
author_node = None
|
||||
try:
|
||||
root = ET.fromstring(context_xml)
|
||||
for node in root.iter("node"):
|
||||
res_id = node.attrib.get("resource-id", "")
|
||||
if "row_feed_photo_profile_name" in res_id or "clips_author_username" in res_id or "profile_header_name" in res_id:
|
||||
author_node = {"original_attribs": node.attrib}
|
||||
logger.debug(f"Identified author_node via structural ID: {res_id}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"XML parse error in author structural fast-path: {e}")
|
||||
|
||||
# 🛡️ Anti-Hallucination Guard: The author header is always near the top. Ignore names in the comment section.
|
||||
if author_node and author_node.get("y", 0) < 1000 and author_node.get("original_attribs", {}).get("text"):
|
||||
result["username"] = author_node["original_attribs"]["text"].strip()
|
||||
# Fallback to Telepathic Engine if structural ID is missing
|
||||
if not author_node:
|
||||
author_node = telepath.find_best_node(
|
||||
context_xml, "post author username text (exclude bottom tabs)", min_confidence=0.75, device=device
|
||||
)
|
||||
logger.debug(f"Telepathic fallback for author_node: {author_node}")
|
||||
|
||||
# 🛡️ Anti-Hallucination Guard: Ensure we actually found text.
|
||||
if author_node:
|
||||
attribs = author_node.get("original_attribs", {})
|
||||
text = attribs.get("text", "").strip()
|
||||
desc = attribs.get("content_desc", "").strip()
|
||||
|
||||
if text:
|
||||
result["username"] = text
|
||||
elif desc:
|
||||
result["username"] = desc
|
||||
else:
|
||||
# If the VLM selected a container (like clips_author_info_component),
|
||||
# extract text from its children.
|
||||
logger.debug("Author node lacks text/desc. Searching children for username...")
|
||||
bounds = attribs.get("bounds")
|
||||
if bounds:
|
||||
try:
|
||||
# Re-parse to find children within bounds
|
||||
import re
|
||||
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
|
||||
if match:
|
||||
l, t, r, b = map(int, match.groups())
|
||||
|
||||
# Fallback: scan all nodes in XML and see if they are inside these bounds
|
||||
possible_texts = []
|
||||
possible_descs = []
|
||||
for n in ET.fromstring(context_xml).iter("node"):
|
||||
child_bounds = n.attrib.get("bounds")
|
||||
child_text = n.attrib.get("text", "").strip()
|
||||
child_desc = n.attrib.get("content-desc", "").strip()
|
||||
|
||||
if child_bounds and (child_text or child_desc):
|
||||
cm = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", child_bounds)
|
||||
if cm:
|
||||
cl, ct, cr, cb = map(int, cm.groups())
|
||||
# Check if child is strictly inside the container
|
||||
if cl >= l and ct >= t and cr <= r and cb <= b:
|
||||
if child_text:
|
||||
possible_texts.append(child_text)
|
||||
if child_desc and "Profile picture" not in child_desc:
|
||||
possible_descs.append(child_desc)
|
||||
|
||||
if possible_texts:
|
||||
result["username"] = possible_texts[0]
|
||||
logger.debug(f"Extracted username '{result['username']}' from child node text.")
|
||||
elif possible_descs:
|
||||
result["username"] = possible_descs[0]
|
||||
logger.debug(f"Extracted username '{result['username']}' from child node desc.")
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to extract username from children: {e}")
|
||||
|
||||
# 2. Learn/extract post media description dynamically
|
||||
media_node = telepath.find_best_node(context_xml, "post media content", min_confidence=0.35)
|
||||
if media_node and media_node.get("original_attribs", {}).get("desc"):
|
||||
result["description"] = media_node["original_attribs"]["desc"].strip()
|
||||
media_node = telepath.find_best_node(
|
||||
context_xml,
|
||||
"post media content (the actual image or video, exclude bottom tabs)",
|
||||
min_confidence=0.35,
|
||||
device=device,
|
||||
)
|
||||
if media_node and media_node.get("original_attribs", {}).get("content_desc"):
|
||||
result["description"] = media_node["original_attribs"]["content_desc"].strip()
|
||||
|
||||
# 3. Visible caption text (heuristic fallback if node isn't explicitly found)
|
||||
# Search all nodes for text that contains the username to find the caption body
|
||||
@@ -85,6 +156,11 @@ def extract_post_content(context_xml: str) -> dict:
|
||||
except Exception as e:
|
||||
logger.warning(f"Error extracting post content autonomously: {e}")
|
||||
|
||||
# REGRESSION FIX 2026-05-01: Flag unreliable data when username is empty
|
||||
if not result["username"]:
|
||||
result["username_missing"] = True
|
||||
logger.warning("⚠️ [PostDataExtraction] Username is empty — data may be unreliable.")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -1,113 +1,808 @@
|
||||
from typing import List, Optional
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
# Navigation tab intent → resource_id keyword mapping
|
||||
_NAV_TAB_MAP = {
|
||||
"tap home tab": "feed_tab",
|
||||
"tap explore tab": "search_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap profile tab": "profile_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _humanize_desc(desc: str) -> str:
|
||||
"""
|
||||
Inserts a space between numbers and letters to fix Instagram's concatenated content-desc.
|
||||
Example: "991following" -> "991 following", "140Kfollowers" -> "140K followers"
|
||||
"""
|
||||
if not desc:
|
||||
return ""
|
||||
import re
|
||||
|
||||
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", desc)
|
||||
|
||||
|
||||
class IntentResolver:
|
||||
"""
|
||||
Translates natural language intents into spatial constraints and node filtering.
|
||||
Replaces the generic text/regex matching with structural intelligence.
|
||||
Vision-First Intent Resolver.
|
||||
|
||||
Resolves UI intents by SEEING the screen, not by parsing text descriptions.
|
||||
Uses Set-of-Mark (SoM) visual prompting: annotates a screenshot with numbered
|
||||
bounding boxes around clickable candidates, sends the annotated image to the VLM,
|
||||
and lets the VLM visually decide which box to tap.
|
||||
|
||||
Architecture:
|
||||
1. Navigation tabs → structural zone guard (bottom 15%, resource-id)
|
||||
2. Everything else → Visual Discovery (screenshot + numbered boxes + VLM)
|
||||
3. Fallback → text-based VLM (when no device/screenshot available)
|
||||
"""
|
||||
|
||||
def resolve(
|
||||
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
"""
|
||||
Finds the best matching node for a given intent autonomously.
|
||||
# ──────────────────────────────────────────────
|
||||
# Structural Guards
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
Navigation tab intents use a structural Zone Guard (bottom 15% of screen)
|
||||
to guarantee we click the actual nav bar, not a content-area element.
|
||||
All other intents delegate to VLM resolution.
|
||||
def filter_navigation_conflicts(
|
||||
self, candidates: List[SpatialNode], intent_description: str, screen_height: int = 2400
|
||||
) -> List[SpatialNode]:
|
||||
"""
|
||||
Prevents VLM from confusing navigation-bar buttons (Back, Close)
|
||||
with bottom tab-bar buttons (Home, Profile, Search).
|
||||
|
||||
Production bug 2026-04-30: VLM picked action_bar_button_back
|
||||
for "tap profile tab" → account switch failed.
|
||||
|
||||
Production bug 2026-05-01: VLM picked profile_tab (desc='Profile')
|
||||
for "post author username text" → navigated to own profile instead.
|
||||
|
||||
Rules:
|
||||
- For tab intents: exclude nodes with "back" in resource_id or
|
||||
content_desc == "Back"
|
||||
- For back/close intents: no filtering (Back is the correct target)
|
||||
- For author/username intents: exclude bottom navigation tabs
|
||||
"""
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
# Only apply for REAL tab navigation intents.
|
||||
# REGRESSION FIX 2026-05-02: "tab" as a substring was too broad.
|
||||
# Intent "post author username text (exclude bottom tabs)" matched
|
||||
# because it contained "tab" → Tab Height Guard nuked the author node.
|
||||
# Now we require specific tab navigation patterns:
|
||||
# - "tap profile tab", "tap home tab", "explore tab"
|
||||
# - NOT "exclude bottom tabs", "tabbar", random mentions
|
||||
import re
|
||||
|
||||
_TAB_PATTERN = re.compile(
|
||||
r"\btap\s+\w+\s+tab\b" # "tap profile tab", "tap home tab"
|
||||
r"|\b\w+\s+tab\b" # "profile tab", "explore tab"
|
||||
r"|^tab\b", # "tab" at start of intent
|
||||
re.IGNORECASE,
|
||||
)
|
||||
filtered = []
|
||||
is_tab_intent = bool(_TAB_PATTERN.search(intent_lower)) and "back" not in intent_lower
|
||||
is_create_intent = "create" in intent_lower or "camera" in intent_lower or "story" in intent_lower
|
||||
# REGRESSION FIX 2026-05-01: Author/username intents must never pick nav tabs
|
||||
is_author_intent = any(kw in intent_lower for kw in ["author", "username", "post media"])
|
||||
|
||||
# Known bottom navigation tab resource_id suffixes
|
||||
NAV_TAB_SUFFIXES = ("_tab", "tab_icon", "navigation_bar")
|
||||
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
|
||||
is_back = "back" in rid or desc == "back"
|
||||
is_close = "close" in rid or desc == "close"
|
||||
is_create = "camera" in rid or "create" in rid or desc == "camera" or desc == "create" or "creation" in rid
|
||||
is_nav_tab = any(rid.endswith(s) for s in NAV_TAB_SUFFIXES)
|
||||
|
||||
if is_tab_intent and (is_back or is_close):
|
||||
logger.debug(
|
||||
f"🛡️ [Nav Conflict Guard] Excluded '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX 2026-05-01: Tab intents must never pick top-screen elements or headers
|
||||
# UPDATE: Actually, tabs are always at the very bottom. Filter out anything above 85% of screen height.
|
||||
if is_tab_intent:
|
||||
is_not_at_bottom = node.center_y < (screen_height * 0.85)
|
||||
if is_not_at_bottom:
|
||||
logger.debug(
|
||||
f"🛡️ [Tab Height Guard] Excluded non-bottom element '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}', y={node.center_y}) for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
# Tab intents should NEVER be content items
|
||||
if any(kw in desc for kw in ["reel by", "photo by", "photos by", "row ", "column "]):
|
||||
logger.debug(
|
||||
f"🛡️ [Content Tab Guard] Excluded content item '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
if is_author_intent and is_nav_tab:
|
||||
logger.debug(
|
||||
f"🛡️ [Author Tab Guard] Excluded nav tab '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
if not is_create_intent and is_create:
|
||||
logger.debug(
|
||||
f"🛡️ [Creation Conflict Guard] Excluded '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude interaction buttons (comment, like, share) when looking for author or media
|
||||
# This prevents the weak VLM from hallucinating bounding box numbers that point to "Comment".
|
||||
interaction_suffixes = ["comment", "like", "share", "send", "save", "button_icon"]
|
||||
is_interaction = any(s in rid for s in interaction_suffixes) or any(s in desc for s in interaction_suffixes)
|
||||
node_text_lower = (node.text or "").lower()
|
||||
is_follow = "follow" in rid or "follow" in node_text_lower or "follow" in desc
|
||||
is_media_intent = "media content" in intent_lower or "image" in intent_lower or "video" in intent_lower
|
||||
|
||||
if is_author_intent and (is_interaction or is_follow):
|
||||
logger.debug(
|
||||
f"🛡️ [Author Interaction Guard] Excluded interaction/follow button '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}', text='{node.text}') for author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
if is_media_intent and (is_interaction or is_follow):
|
||||
logger.debug(
|
||||
f"🛡️ [Media Interaction Guard] Excluded interaction/follow button '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for media intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude stories and reels tray when looking for a POST author
|
||||
# This prevents the VLM from selecting the user's own story at the top of the feed
|
||||
is_post_author_intent = is_author_intent and "post" in intent_lower
|
||||
node_text_lower = (node.text or "").lower()
|
||||
is_story_or_reel = (
|
||||
"story" in rid
|
||||
or "story" in desc
|
||||
or "story" in node_text_lower
|
||||
or "reel" in rid
|
||||
or "reel" in desc
|
||||
or "reel" in node_text_lower
|
||||
)
|
||||
|
||||
if is_post_author_intent and is_story_or_reel:
|
||||
logger.debug(
|
||||
f"🛡️ [Post Author Story Guard] Excluded story/reel '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for post author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude action bar titles (like 'For you') when looking for an author
|
||||
if is_author_intent and "action_bar_title" in rid:
|
||||
logger.debug(
|
||||
f"🛡️ [Author Action Bar Guard] Excluded action bar title '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
filtered.append(node)
|
||||
|
||||
return filtered
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Public API
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def resolve(
|
||||
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
# ── Navigation Bar Zone Guard ──
|
||||
# When intent targets a nav tab, resolve structurally to the bottom nav zone.
|
||||
# This prevents the VLM from selecting content profile pictures instead of tabs.
|
||||
# The bottom navigation bar is always in the bottom 15% of the screen.
|
||||
tab_keyword = _NAV_TAB_MAP.get(intent_lower)
|
||||
if tab_keyword:
|
||||
nav_zone_y = int(screen_height * 0.85)
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower()
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
# Fallback: broader search in nav zone by content_desc
|
||||
tab_label = intent_lower.replace("tap ", "").replace(" tab", "")
|
||||
nav_candidates = [
|
||||
n for n in candidates if n.y1 >= nav_zone_y and tab_label in (n.content_desc or "").lower()
|
||||
]
|
||||
if nav_candidates:
|
||||
return nav_candidates[0]
|
||||
return None
|
||||
|
||||
# If the intent is a high-level GOAL that accidentally leaked into the IntentResolver,
|
||||
# we explicitly block it from clicking random nodes.
|
||||
# IMPORTANT: Use exact match to avoid blocking "tap profile tab" when filtering "open profile"
|
||||
# Block abstract goals from leaking into node clicks
|
||||
abstract_goals = ["open profile", "open explore", "open following", "learn own profile"]
|
||||
if intent_lower in abstract_goals:
|
||||
return None
|
||||
|
||||
# 1. Ask the Telepathic VLM to find the best node
|
||||
import json
|
||||
# --- Strict Structural Fast-Paths ---
|
||||
# Bypass VLM for deterministically identifiable UI components
|
||||
if "message text box" in intent_lower or "message input" in intent_lower or "type message" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "composer_edittext" in rid or "message…" in text or "message..." in text:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
|
||||
return node
|
||||
|
||||
if "last received message text" in intent_lower or "received message" in intent_lower:
|
||||
# Gather all message text views
|
||||
msg_nodes = [n for n in candidates if "direct_text_message_text_view" in (n.resource_id or "").lower()]
|
||||
if msg_nodes:
|
||||
# The last one in the XML is typically the most recent message at the bottom of the screen
|
||||
latest_msg = msg_nodes[-1]
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
|
||||
return latest_msg
|
||||
|
||||
if "send message button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "send" in rid or "composer_button" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
|
||||
return node
|
||||
|
||||
if "post author username" in intent_lower or "tap post username" in intent_lower:
|
||||
for node in candidates:
|
||||
if "row_feed_photo_profile_imageview" in (node.resource_id or "").lower():
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
|
||||
return node
|
||||
for node in candidates:
|
||||
if "row_feed_photo_profile_name" in (node.resource_id or "").lower():
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
|
||||
return node
|
||||
|
||||
if "feed post content" in intent_lower or "post media content" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_photo_imageview" in rid or "zoomable_view_container" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
|
||||
return node
|
||||
|
||||
if "comment" in intent_lower and "button" in intent_lower:
|
||||
# First try the View all comments button
|
||||
for node in candidates:
|
||||
if (
|
||||
"view all comments" in (node.text or "").lower()
|
||||
or "view all comments" in (node.content_desc or "").lower()
|
||||
):
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found comment button text: {node.text or node.content_desc}"
|
||||
)
|
||||
return node
|
||||
# Then try the icon itself if somehow clickable
|
||||
for node in candidates:
|
||||
if (
|
||||
"row_feed_button_comment" in (node.resource_id or "").lower()
|
||||
or "row_feed_textview_comments" in (node.resource_id or "").lower()
|
||||
):
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
|
||||
return node
|
||||
|
||||
if "like" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_like" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found like button: {rid}")
|
||||
return node
|
||||
|
||||
if ("send" in intent_lower or "share" in intent_lower) and "post" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row_feed_button_share" in rid or "send post" in desc:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
return node
|
||||
|
||||
if "add to story" in intent_lower:
|
||||
# We skip structural fast-path for 'add to story' since it relies heavily on language/text strings
|
||||
# and let the VLM figure it out or rely on purely visual indicators.
|
||||
pass
|
||||
|
||||
if "share" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_share" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found share button: {rid}")
|
||||
return node
|
||||
|
||||
if "save" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_save" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found save button: {rid}")
|
||||
return node
|
||||
|
||||
if "follow" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "profile_header_follow_button" in rid or "inline_follow_button" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
|
||||
return node
|
||||
|
||||
if "first post" in intent_lower or "first item" in intent_lower or "first search result" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "grid_card_layout_container" in rid or "image_button" in rid or "row_search_user" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
|
||||
return node
|
||||
|
||||
if "story ring" in intent_lower or "story tray" in intent_lower:
|
||||
story_nodes = []
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
# Instagram story tray avatars usually have this resource id and 'story' in the content description
|
||||
if ("avatar_image_view" in rid or "row_profile_header_imageview" in rid) and "story" in desc:
|
||||
# Ignore the user's explicit "Add to story" ring
|
||||
if "add to story" not in desc and "your story" not in text:
|
||||
story_nodes.append(node)
|
||||
|
||||
if story_nodes:
|
||||
# Sort horizontally (left-to-right)
|
||||
story_nodes.sort(key=lambda n: n.x1)
|
||||
|
||||
# Check if this is the home feed story tray (avatar_image_view without 'highlight' in desc)
|
||||
is_highlight = any("highlight" in (n.content_desc or "").lower() for n in story_nodes)
|
||||
if "avatar_image_view" in (story_nodes[0].resource_id or "").lower() and not is_highlight:
|
||||
if len(story_nodes) > 1:
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found {len(story_nodes)} story rings. Skipping own profile. Picking second: '{story_nodes[1].content_desc}'"
|
||||
)
|
||||
return story_nodes[1]
|
||||
else:
|
||||
logger.warning(
|
||||
"🎯 [Structural Fast-Path] Only 1 story ring found on feed (likely own profile). Skipping to avoid modal trap."
|
||||
)
|
||||
return None
|
||||
else:
|
||||
# Profile header or other single-story views
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found story ring avatar: {story_nodes[0].resource_id} (desc: '{story_nodes[0].content_desc}')"
|
||||
)
|
||||
return story_nodes[0]
|
||||
|
||||
# --- Navigation Tab Fast-Paths ---
|
||||
# Deterministically identify bottom navigation tabs to prevent VLM confusion
|
||||
tab_map = {
|
||||
"home tab": "feed_tab",
|
||||
"feed tab": "feed_tab",
|
||||
"reels tab": "clips_tab",
|
||||
"clips tab": "clips_tab",
|
||||
"explore tab": "search_tab",
|
||||
"search tab": "search_tab",
|
||||
"profile tab": "profile_tab",
|
||||
"message tab": "direct_tab",
|
||||
"direct tab": "direct_tab",
|
||||
}
|
||||
for intent_key, resource_suffix in tab_map.items():
|
||||
if intent_key in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if rid.endswith(f":id/{resource_suffix}"):
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
|
||||
return node
|
||||
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
# we strictly filter candidates to those whose text or content_desc contains the quote.
|
||||
import re
|
||||
|
||||
quotes = re.findall(r"['\"](.*?)['\"]", intent_description)
|
||||
if quotes:
|
||||
target_text = quotes[0].lower()
|
||||
|
||||
# Only use the exact target string (no manual localized translation dictionaries!)
|
||||
localized_targets = [target_text]
|
||||
|
||||
semantic_candidates = []
|
||||
for node in candidates:
|
||||
n_text = (node.text or "").lower()
|
||||
n_desc = (node.content_desc or "").lower()
|
||||
|
||||
# Check if any of the localized targets match
|
||||
for loc_target in localized_targets:
|
||||
pattern = r"\b" + re.escape(loc_target) + r"\b"
|
||||
if re.search(pattern, n_text) or re.search(pattern, n_desc):
|
||||
semantic_candidates.append(node)
|
||||
break # Found a match, no need to check other localized targets
|
||||
|
||||
if semantic_candidates:
|
||||
if len(semantic_candidates) == 1:
|
||||
logger.info(f"🎯 [Semantic Guard] Exact match found for '{target_text}', skipping VLM.")
|
||||
return semantic_candidates[0]
|
||||
else:
|
||||
logger.info(
|
||||
f"🎯 [Semantic Guard] {len(semantic_candidates)} matches found for '{target_text}'. Reducing candidates for VLM."
|
||||
)
|
||||
candidates = semantic_candidates
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ [Semantic Guard] No candidates found containing '{target_text}'. Returning None to prevent hallucination."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── PRIMARY PATH: Visual Discovery ──
|
||||
# If we have a device, the VLM SEES the screen and decides.
|
||||
if device is not None and (
|
||||
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
|
||||
):
|
||||
print(f"DEBUG_INTENT: Entering Visual Discovery for '{intent_description}'")
|
||||
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
|
||||
return self._visual_discovery(intent_description, candidates, device, screen_height=screen_height)
|
||||
|
||||
print(f"DEBUG_INTENT: Falling back to Text-based VLM for '{intent_description}'")
|
||||
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
|
||||
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure.
|
||||
if "following list" in intent_lower or "followers list" in intent_lower or "tap message button" in intent_lower:
|
||||
logger.warning(
|
||||
f"🛡️ [Hallucination Guard] Intent '{intent_description}' is a strict structural target. "
|
||||
"Since it wasn't resolved by fast-paths, it is missing. Rejecting VLM fallback."
|
||||
)
|
||||
return None
|
||||
|
||||
# ── FALLBACK: Text-based VLM resolution ──
|
||||
# Only used when device is unavailable (e.g., unit tests without screenshots).
|
||||
return self._text_based_resolve(intent_description, candidates, device, screen_height=screen_height)
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Visual Discovery (Set-of-Mark Prompting)
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def _annotate_screenshot_with_candidates(
|
||||
self, device, candidates: List[SpatialNode]
|
||||
) -> Tuple[str, Dict[int, SpatialNode]]:
|
||||
"""
|
||||
Takes a screenshot and draws numbered bounding boxes around clickable candidates.
|
||||
|
||||
Returns:
|
||||
annotated_b64: Base64-encoded JPEG of the annotated screenshot.
|
||||
box_map: Dict mapping box number → SpatialNode for coordinate lookup.
|
||||
"""
|
||||
from PIL import ImageDraw
|
||||
|
||||
img = device.deviceV2.screenshot()
|
||||
|
||||
# Stage 1: Basic area filter + exclude system UI and notifications (ALREADY HANDLED in _visual_discovery)
|
||||
pre_filtered = candidates
|
||||
|
||||
# Stage 2: Spatial deduplication
|
||||
# A node could completely contain another.
|
||||
# If parent is clickable and child is not: suppress child (e.g. text inside button)
|
||||
# If parent is not clickable and child is: suppress parent (e.g. layout container around button)
|
||||
# If both are not clickable: suppress parent (keep the smaller, more specific text)
|
||||
# If both are clickable: keep both! (e.g. nested buttons like row and camera icon)
|
||||
def _contains(parent: SpatialNode, child: SpatialNode) -> bool:
|
||||
return (
|
||||
parent.x1 <= child.x1
|
||||
and parent.y1 <= child.y1
|
||||
and parent.x2 >= child.x2
|
||||
and parent.y2 >= child.y2
|
||||
and parent.node_id != child.node_id
|
||||
)
|
||||
|
||||
to_suppress = set()
|
||||
# Sort by area DESCENDING so we process largest (parents) first
|
||||
pre_filtered.sort(key=lambda n: n.area, reverse=True)
|
||||
|
||||
for i, parent in enumerate(pre_filtered):
|
||||
for j in range(i + 1, len(pre_filtered)):
|
||||
child = pre_filtered[j]
|
||||
if _contains(parent, child):
|
||||
if parent.clickable and not child.clickable:
|
||||
to_suppress.add(child.node_id)
|
||||
# Merge semantic info from child to parent if missing
|
||||
if (
|
||||
child.text
|
||||
and child.text not in (parent.text or "")
|
||||
and child.text not in (parent.content_desc or "")
|
||||
):
|
||||
parent.content_desc = f"{(parent.content_desc or '')} {child.text}".strip()
|
||||
if (
|
||||
child.content_desc
|
||||
and child.content_desc not in (parent.text or "")
|
||||
and child.content_desc not in (parent.content_desc or "")
|
||||
):
|
||||
parent.content_desc = f"{(parent.content_desc or '')} {child.content_desc}".strip()
|
||||
elif not parent.clickable and child.clickable:
|
||||
to_suppress.add(parent.node_id)
|
||||
# Pass any semantic info down just in case
|
||||
if parent.content_desc and not child.content_desc:
|
||||
child.content_desc = parent.content_desc
|
||||
if parent.text and not child.text:
|
||||
child.text = parent.text
|
||||
elif not parent.clickable and not child.clickable:
|
||||
to_suppress.add(parent.node_id)
|
||||
if parent.content_desc and not child.content_desc:
|
||||
child.content_desc = parent.content_desc
|
||||
elif parent.clickable and child.clickable:
|
||||
# Keep both, distinct nested interactables
|
||||
pass
|
||||
|
||||
visible_candidates = [n for n in pre_filtered if n.node_id not in to_suppress]
|
||||
|
||||
draw = ImageDraw.Draw(img)
|
||||
box_map: Dict[int, SpatialNode] = {}
|
||||
|
||||
# Color palette for distinct boxes
|
||||
colors = [
|
||||
(255, 0, 0),
|
||||
(0, 200, 0),
|
||||
(0, 0, 255),
|
||||
(255, 165, 0),
|
||||
(128, 0, 128),
|
||||
(0, 200, 200),
|
||||
(255, 20, 147),
|
||||
(0, 128, 0),
|
||||
(255, 215, 0),
|
||||
(70, 130, 180),
|
||||
]
|
||||
|
||||
for i, node in enumerate(visible_candidates):
|
||||
color = colors[i % len(colors)]
|
||||
|
||||
# Draw bounding box
|
||||
draw.rectangle(
|
||||
[node.x1, node.y1, node.x2, node.y2],
|
||||
outline=color,
|
||||
width=3,
|
||||
)
|
||||
|
||||
# Draw number label with background for readability
|
||||
label = str(i)
|
||||
label_x = node.x1 + 2
|
||||
label_y = max(node.y1 - 18, 0)
|
||||
|
||||
# Draw label background
|
||||
bbox = draw.textbbox((label_x, label_y), label)
|
||||
draw.rectangle(
|
||||
[bbox[0] - 2, bbox[1] - 2, bbox[2] + 2, bbox[3] + 2],
|
||||
fill=color,
|
||||
)
|
||||
draw.text((label_x, label_y), label, fill=(255, 255, 255))
|
||||
|
||||
box_map[i] = node
|
||||
|
||||
# Encode to base64 JPEG
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="JPEG", quality=85)
|
||||
annotated_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
return annotated_b64, box_map
|
||||
|
||||
def _visual_discovery(
|
||||
self, intent_description: str, candidates: List[SpatialNode], device, screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
"""
|
||||
Vision-first intent resolution via Set-of-Mark (SoM) prompting.
|
||||
|
||||
1. Takes a screenshot
|
||||
2. Draws numbered bounding boxes on clickable candidates
|
||||
3. Sends the annotated screenshot to the VLM
|
||||
4. VLM SEES the UI and picks which numbered box matches the intent
|
||||
5. Maps box number back to SpatialNode for precise coordinates
|
||||
"""
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
# Pre-filter candidates to reduce VLM hallucinations
|
||||
filtered_candidates = []
|
||||
for n in candidates:
|
||||
# Skip massive background containers
|
||||
if n.area > 500000:
|
||||
continue
|
||||
# Pre-filter candidates by area and system UI before any semantic matching
|
||||
candidates = [
|
||||
n
|
||||
for n in candidates
|
||||
if 200 < n.area < 400000
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# Structural heuristic: if looking for profile, prioritize nodes that might be profiles
|
||||
# and exclude obvious bottom tabs/navigation
|
||||
if "profile" in intent_lower:
|
||||
res = (n.resource_id or "").lower()
|
||||
if "tab" in res or "navigation" in res or "action_bar" in res:
|
||||
continue
|
||||
filtered_candidates.append(n)
|
||||
# --- Navigation Conflict Guard ---
|
||||
# Prevents VLM from confusing Back buttons with tab buttons
|
||||
# Production bug 2026-04-30: VLM picked Back for "tap profile tab"
|
||||
candidates = self.filter_navigation_conflicts(candidates, intent_description, screen_height=screen_height)
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
# to prevent the VLM from hallucinating text nodes as interactive buttons.
|
||||
intent_lower = intent_description.lower()
|
||||
if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower:
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
text_len = len(node.text or "")
|
||||
if text_len < 40:
|
||||
filtered_candidates.append(node)
|
||||
else:
|
||||
logger.debug(f"🛡️ [Strict Button Guard] Filtered out node with long text: '{node.text[:20]}...'")
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Post/Grid Item Guard ---
|
||||
# VLMs frequently hallucinate 'Search' when asked to tap a post. We must pre-filter.
|
||||
if "first post" in intent_lower or "grid item" in intent_lower:
|
||||
grid_candidates = []
|
||||
for node in candidates:
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Posts/grid items usually have 'row X, column Y', 'photos by', or 'reel by'
|
||||
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
|
||||
grid_candidates.append(node)
|
||||
|
||||
if grid_candidates:
|
||||
logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.")
|
||||
candidates = grid_candidates
|
||||
|
||||
# --- Author/Username Guard ---
|
||||
# Prevents VLM from picking the "Profile" nav tab when asked for "post author username".
|
||||
if "author" in intent_lower or "username" in intent_lower or "profile name" in intent_lower:
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
res_id = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if (
|
||||
"tab" in res_id
|
||||
or "navigation" in res_id
|
||||
or "tabbar" in res_id
|
||||
or desc in ["home", "search", "reels", "profile"]
|
||||
):
|
||||
logger.debug(
|
||||
f"🛡️ [Author Guard] Filtered out navigation tab: '{node.content_desc}' ({node.resource_id})"
|
||||
)
|
||||
else:
|
||||
filtered_candidates.append(node)
|
||||
candidates = filtered_candidates
|
||||
|
||||
try:
|
||||
annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
logger.warning(f"⚠️ [Visual Discovery] Screenshot annotation failed: {e}")
|
||||
return None
|
||||
|
||||
if not box_map:
|
||||
return None
|
||||
|
||||
self.last_box_map = box_map
|
||||
|
||||
cfg = Config()
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
# Build a compact legend of what each box contains
|
||||
box_legend_lines = []
|
||||
for idx in sorted(box_map.keys()):
|
||||
node = box_map[idx]
|
||||
label_parts = []
|
||||
if node.content_desc:
|
||||
desc = _humanize_desc(node.content_desc)
|
||||
label_parts.append(f"desc='{desc[:50]}'")
|
||||
if node.text and node.text != node.content_desc:
|
||||
text = _humanize_desc(node.text)
|
||||
label_parts.append(f"text='{text[:50]}'")
|
||||
if not label_parts:
|
||||
label_parts.append("(no visible text)")
|
||||
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
|
||||
box_legend = "\n".join(box_legend_lines)
|
||||
logger.debug(f"BOX LEGEND:\n{box_legend}")
|
||||
|
||||
prompt = (
|
||||
f"You are looking at a mobile app screenshot with numbered bounding boxes drawn around interactive UI elements.\n"
|
||||
f"Each box has a number label in a colored rectangle.\n\n"
|
||||
f"Box legend (what each box contains):\n{box_legend}\n\n"
|
||||
f"Your task: Find the exact box number that corresponds to this intent: '{intent_description}'\n\n"
|
||||
f"CRITICAL RULES:\n"
|
||||
f"1. If the intent contains a word in quotes (e.g., 'Search', 'New Message'), you MUST look at the Box legend and pick the box that contains that word (case-insensitive). Do not pick anything else.\n"
|
||||
f"2. For icons without text:\n"
|
||||
f" - 'like button' = HEART-SHAPED ICON (♡/❤), usually has desc='Like'.\n"
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
|
||||
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
|
||||
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
|
||||
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
|
||||
f" - Do NOT pick navigation buttons like 'Search'.\n"
|
||||
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f" - These are always at the BOTTOM edge of the screen.\n"
|
||||
f" - 'profile tab' is usually the furthest right icon (your avatar).\n"
|
||||
f" - 'home tab' is the furthest left icon (house).\n"
|
||||
f" - 'explore tab' is the magnifying glass.\n"
|
||||
f" - 'reels tab' is the video clapperboard.\n"
|
||||
f"8. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
|
||||
f"9. If the intent is 'save post':\n"
|
||||
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
|
||||
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f"10. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n"
|
||||
f" - Bottom Navigation Tabs (Home, Search, Reels, Profile) are ALWAYS at the very bottom (y > 2100).\n"
|
||||
f" - Content Interaction Buttons (Like, Comment, Share, Reactions, Message Input) are attached to posts or threads, NOT the bottom nav bar.\n"
|
||||
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
|
||||
f"11. If the intent is 'feed post content' or 'post media content':\n"
|
||||
f" - Pick the largest box that contains the actual image or video, usually described as 'Photo', 'Video', or 'Carousel'.\n"
|
||||
f"12. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
try:
|
||||
res = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict visual JSON box selector. Respond only with JSON.",
|
||||
user_prompt=prompt,
|
||||
use_local_edge=True,
|
||||
images_b64=[annotated_b64],
|
||||
)
|
||||
print(f"DEBUG_INTENT: VLM RAW RESPONSE for '{intent_description}': {res}")
|
||||
data = json.loads(res)
|
||||
box_idx = data.get("box")
|
||||
if box_idx is None:
|
||||
box_idx = data.get("selected_index")
|
||||
if box_idx is None:
|
||||
box_idx = data.get("box_index")
|
||||
if box_idx is None:
|
||||
box_idx = data.get("index")
|
||||
|
||||
if box_idx is not None and box_idx in box_map:
|
||||
selected = box_map[box_idx]
|
||||
logger.info(
|
||||
f"👁️ [Visual Discovery] VLM selected box [{box_idx}] → "
|
||||
f"id='{selected.resource_id}', desc='{selected.content_desc}'"
|
||||
)
|
||||
return selected
|
||||
else:
|
||||
logger.warning(
|
||||
f"👁️ [Visual Discovery] VLM returned box={box_idx} which is not in box_map ({list(box_map.keys())[:5]}...)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [Visual Discovery] VLM call failed: {e}")
|
||||
|
||||
return None
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Text-based Fallback (no device/screenshot)
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def _text_based_resolve(
|
||||
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
|
||||
) -> Optional[SpatialNode]:
|
||||
"""
|
||||
Fallback resolution via text descriptions of XML nodes.
|
||||
Used only when no device is available for screenshots.
|
||||
"""
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
filtered_candidates = [n for n in candidates if n.area < 500000]
|
||||
filtered_candidates = self.filter_navigation_conflicts(
|
||||
filtered_candidates, intent_description, screen_height=screen_height
|
||||
)
|
||||
if "profile" in intent_lower:
|
||||
filtered_candidates = [
|
||||
n
|
||||
for n in filtered_candidates
|
||||
if not any(kw in (n.resource_id or "").lower() for kw in ("tab", "navigation", "action_bar"))
|
||||
]
|
||||
if not filtered_candidates:
|
||||
filtered_candidates = candidates
|
||||
filtered_candidates = [n for n in candidates if n.area < 500000]
|
||||
|
||||
cfg = Config()
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
# Prepare context
|
||||
node_context = []
|
||||
for i, node in enumerate(filtered_candidates):
|
||||
text = node.text or ""
|
||||
desc = node.content_desc or ""
|
||||
text = _humanize_desc(node.text or "")
|
||||
desc = _humanize_desc(node.content_desc or "")
|
||||
res_id = node.resource_id or ""
|
||||
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")
|
||||
|
||||
prompt = (
|
||||
f"You are a Spatial UI Intent Resolver.\n"
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n"
|
||||
f"CRITICAL RULES:\n"
|
||||
f"- If the intent is about opening the 'post author', STRICTLY require 'row_feed_photo_profile' in the ID. Do not select comment authors.\n"
|
||||
f"- If the intent is about opening a user profile generally, prioritize nodes containing 'profile_name' or 'profile_image' in their ID, NOT generic action bars or tabs.\n"
|
||||
f"- Ignore bottom navigation tabs (home, search, profile) UNLESS the intent explicitly asks to navigate to a primary feed.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"CRITICAL RULES:\n"
|
||||
"1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
" - These are always at the BOTTOM of the screen (typically y > 2100).\n"
|
||||
" - 'profile tab' is usually the furthest right.\n"
|
||||
" - 'home tab' is the furthest left.\n"
|
||||
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
|
||||
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
"If none of the candidates match the intent, return null."
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -118,13 +813,12 @@ class IntentResolver:
|
||||
user_prompt=prompt,
|
||||
use_local_edge=True,
|
||||
)
|
||||
print(f"DEBUG_INTENT: TEXT LLM RAW RESPONSE for '{intent_description}': {res}")
|
||||
data = json.loads(res)
|
||||
idx = data.get("selected_index")
|
||||
if idx is not None and 0 <= idx < len(filtered_candidates):
|
||||
return filtered_candidates[idx]
|
||||
except Exception as e:
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).warning(f"⚠️ [IntentResolver] VLM resolution failed ({e}).")
|
||||
logger.warning(f"⚠️ [IntentResolver] Text-based VLM resolution failed ({e}).")
|
||||
|
||||
return None
|
||||
|
||||
@@ -43,7 +43,7 @@ class ScreenIdentity:
|
||||
except ImportError:
|
||||
self.screen_memory = None
|
||||
|
||||
def identify(self, xml_dump: str) -> Dict[str, Any]:
|
||||
def identify(self, xml_dump: str, screenshot_b64: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Analyzes an XML dump and returns a complete screen description.
|
||||
|
||||
@@ -116,6 +116,11 @@ class ScreenIdentity:
|
||||
}
|
||||
)
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
sae = SituationalAwarenessEngine.get_instance()
|
||||
signature = sae._compress_xml(xml_dump) if sae else self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Foreign app check ──
|
||||
if app_id not in packages:
|
||||
return {
|
||||
@@ -123,18 +128,16 @@ class ScreenIdentity:
|
||||
"available_actions": ["press back", "force start instagram"],
|
||||
"selected_tab": None,
|
||||
"context": {"packages": list(packages)},
|
||||
"signature": self._compute_signature(resource_ids, content_descs, texts),
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
desc_lower = " ".join(content_descs).lower()
|
||||
text_lower = " ".join(texts).lower()
|
||||
ids_str = " ".join(resource_ids).lower()
|
||||
|
||||
signature = self._compute_signature(resource_ids, content_descs, texts)
|
||||
|
||||
# ── Identify screen type from structural signals ──
|
||||
screen_type = self._classify_screen(
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature
|
||||
resource_ids, content_descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature, screenshot_b64
|
||||
)
|
||||
|
||||
# ── Extract available actions from clickable elements ──
|
||||
@@ -153,32 +156,45 @@ class ScreenIdentity:
|
||||
"signature": signature,
|
||||
}
|
||||
|
||||
def _classify_screen(self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None):
|
||||
"""Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
def _classify_screen(
|
||||
self, ids, descs, texts, selected_tab, desc_lower, text_lower, ids_str, signature=None, screenshot_b64=None
|
||||
):
|
||||
"""
|
||||
Classify screen type using Semantic Memory with LLM fallback — NO hardcoded states."""
|
||||
|
||||
# Priority 0: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 1: Check Qdrant Semantic Cache
|
||||
# Priority 0: Fetch Qdrant Semantic Cache
|
||||
# We fetch this early to see if there is a 'NORMAL' override for the MODAL check.
|
||||
# We DO NOT let this override deterministic structural heuristics! Fuzzy vector matching
|
||||
# can easily confuse HOME_FEED and OWN_PROFILE if the bottom navigation bar is identical.
|
||||
cached_type_str = None
|
||||
if signature and self.screen_memory and self.screen_memory.is_connected:
|
||||
cached_type_str = self.screen_memory.get_screen_type(signature, similarity_threshold=0.92)
|
||||
if cached_type_str:
|
||||
try:
|
||||
return ScreenType[cached_type_str]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Priority 2: Structural Heuristics (Instant, for core tabs)
|
||||
is_normal_override = cached_type_str == "NORMAL"
|
||||
|
||||
# Priority 1: Content-creation overlays that block ALL navigation.
|
||||
# These full-screen Instagram UIs have no navigation tabs and trap the bot.
|
||||
# Structural detection is O(1), zero LLM calls, and cannot be fooled.
|
||||
if not is_normal_override:
|
||||
creation_flow_markers = ("quick_capture", "gallery_cancel_button", "creation_flow", "reel_camera")
|
||||
if any(marker in ids_str for marker in creation_flow_markers):
|
||||
logger.info("🛡️ [ScreenIdentity] Content-creation overlay detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 2: Structural Heuristics (100% Deterministic)
|
||||
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
|
||||
return ScreenType.FOLLOW_LIST
|
||||
|
||||
if "profile_header_container" in ids:
|
||||
if selected_tab == "profile_tab":
|
||||
# Profile structural markers
|
||||
PROFILE_MARKERS = (
|
||||
"profile_header_container",
|
||||
"row_profile_header_imageview",
|
||||
"profile_tabs_container",
|
||||
"profile_header_name",
|
||||
)
|
||||
if any(marker in ids for marker in PROFILE_MARKERS):
|
||||
own_profile_texts = ("edit profile", "share profile", "profil bearbeiten", "profil teilen")
|
||||
if selected_tab == "profile_tab" or any(m in desc_lower or m in text_lower for m in own_profile_texts):
|
||||
return ScreenType.OWN_PROFILE
|
||||
return ScreenType.OTHER_PROFILE
|
||||
|
||||
@@ -188,12 +204,36 @@ class ScreenIdentity:
|
||||
if any(marker in ids for marker in REELS_MARKERS):
|
||||
return ScreenType.REELS_FEED
|
||||
|
||||
# DM thread detection — structural markers present inside DM conversations
|
||||
if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids:
|
||||
# DM thread detection — Semantic app-agnostic markers (chat input fields)
|
||||
chat_input_markers = ["Message...", "Nachricht...", "Type a message", "Nachricht senden", "Send a message"]
|
||||
if any(marker in texts for marker in chat_input_markers) or "direct_thread_header" in ids:
|
||||
return ScreenType.DM_THREAD
|
||||
|
||||
if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab:
|
||||
return ScreenType.POST_DETAIL
|
||||
# POST_DETAIL vs HOME_FEED: Both have row_feed_* markers. The differentiator
|
||||
# is that HOME_FEED has the main_feed_action_bar (top bar with 'Instagram' title).
|
||||
# POST_DETAIL lacks this because it shows a single expanded post.
|
||||
# Note: We MUST NOT use `not selected_tab` here — posts opened from feed
|
||||
# retain the feed_tab as selected, which previously caused misclassification.
|
||||
if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids:
|
||||
if "main_feed_action_bar" not in ids:
|
||||
return ScreenType.POST_DETAIL
|
||||
|
||||
# Story view structural markers — present in full-screen story viewer.
|
||||
# Stories hide the navigation tab bar, so selected_tab is always None.
|
||||
# Must be checked BEFORE tab-based fallbacks to prevent UNKNOWN classification.
|
||||
STORY_MARKERS = (
|
||||
"reel_viewer_media_layout",
|
||||
"reel_viewer_header",
|
||||
"reel_viewer_progress_bar",
|
||||
"reel_viewer_root",
|
||||
"story_viewer_container",
|
||||
"reel_viewer_content_layout",
|
||||
)
|
||||
if any(marker in ids for marker in STORY_MARKERS):
|
||||
return ScreenType.STORY_VIEW
|
||||
# Fallback: content-desc "Like Story" or "Send story" confirms story context
|
||||
if "like story" in desc_lower or "send story" in desc_lower or "nachricht senden" in desc_lower:
|
||||
return ScreenType.STORY_VIEW
|
||||
|
||||
if selected_tab == "feed_tab":
|
||||
return ScreenType.HOME_FEED
|
||||
@@ -201,6 +241,8 @@ class ScreenIdentity:
|
||||
return ScreenType.REELS_FEED
|
||||
if selected_tab == "search_tab":
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if "action_bar_search_edit_text" in ids:
|
||||
return ScreenType.EXPLORE_GRID
|
||||
if selected_tab == "profile_tab":
|
||||
return ScreenType.OWN_PROFILE
|
||||
if selected_tab == "direct_tab":
|
||||
@@ -208,41 +250,75 @@ class ScreenIdentity:
|
||||
if "message_input" in ids:
|
||||
return ScreenType.DM_INBOX # Fallback for DM thread as inbox
|
||||
|
||||
# Priority 3: Semantic VLM Classification Fallback
|
||||
# End of structural heuristics
|
||||
|
||||
# Priority 3: Cached Semantic Type (If deterministic heuristics failed)
|
||||
if cached_type_str and cached_type_str != "NORMAL":
|
||||
try:
|
||||
cached_type = ScreenType[cached_type_str]
|
||||
# Enforce absolute structural parity: Story and Reels must have their structural markers.
|
||||
# If they reached Priority 3, it means Priority 2 failed to find their markers.
|
||||
# Therefore, any cache telling us this is a Story/Reel without those markers is hallucinating.
|
||||
if cached_type in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
|
||||
logger.warning(
|
||||
f"⚠️ [ScreenIdentity] Rejecting cached {cached_type.name} due to missing structural markers."
|
||||
)
|
||||
else:
|
||||
return cached_type
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Priority 4: Semantic VLM Classification Fallback
|
||||
if not screenshot_b64 and getattr(self, "device", None) is not None:
|
||||
screenshot_b64 = self.device.get_screenshot_b64()
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
cfg = Config()
|
||||
url = (
|
||||
getattr(cfg.args, "ai_embedding_url", "http://localhost:11434/api/chat")
|
||||
getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/chat"
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_embedding_model", "llama3") if hasattr(cfg, "args") else "llama3"
|
||||
model = getattr(cfg.args, "ai_telepathic_model", "llava:latest") if hasattr(cfg, "args") else "llava:latest"
|
||||
|
||||
layout_context = (
|
||||
f"Selected Tab: {selected_tab}\nResource IDs: {list(ids)}\nVisible Texts context: {texts[:10]}\n"
|
||||
)
|
||||
prompt = (
|
||||
f"Identify the Instagram screen layout type based on these DOM structural signals.\n"
|
||||
f"Identify the Instagram screen layout type based on the provided screenshot and structural signals.\n"
|
||||
f"Valid types: {[t.name for t in ScreenType]}\n"
|
||||
f"Context:\n{layout_context}\n"
|
||||
f"Reply ONLY with the exact matching enum Type Name string, or 'UNKNOWN' if no type matches."
|
||||
)
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Classify this screen layout.", system=prompt, format_json=False
|
||||
response = query_telepathic_llm(
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt=prompt,
|
||||
user_prompt="Classify this screen layout.",
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
use_local_edge=True,
|
||||
)
|
||||
if response and isinstance(response, str):
|
||||
result = response.strip().upper()
|
||||
elif response and isinstance(response, dict) and "response" in response:
|
||||
result = response["response"].strip().upper()
|
||||
else:
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
result = response.strip().upper() if response else "UNKNOWN"
|
||||
|
||||
for t in ScreenType:
|
||||
if t.name in result:
|
||||
if is_normal_override and t == ScreenType.MODAL:
|
||||
# Prevent the LLM from hallucinating an obstacle if explicitly verified as NORMAL
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
# Enforce absolute structural parity: Story and Reels must have their structural markers.
|
||||
if t in (ScreenType.STORY_VIEW, ScreenType.REELS_FEED):
|
||||
logger.warning(
|
||||
f"⚠️ [ScreenIdentity] Rejecting VLM hallucinated {t.name} due to missing structural markers."
|
||||
)
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
if signature and self.screen_memory:
|
||||
self.screen_memory.store_screen(signature, t.name)
|
||||
return t
|
||||
@@ -283,7 +359,18 @@ class ScreenIdentity:
|
||||
actions.append("tap save button")
|
||||
if "back" in desc_lower:
|
||||
actions.append("tap back button")
|
||||
if any("follow" in e.get("text", "").lower() for e in clickable_elements):
|
||||
has_following = any(
|
||||
"following" in e.get("text", "").lower() or "following" in e.get("desc", "").lower()
|
||||
for e in clickable_elements
|
||||
)
|
||||
if has_following:
|
||||
actions.append("tap following button")
|
||||
elif any(
|
||||
"follow" in e.get("text", "").lower()
|
||||
or "follow" in e.get("desc", "").lower()
|
||||
or "follow" in e.get("id", "").lower()
|
||||
for e in clickable_elements
|
||||
):
|
||||
actions.append("tap follow button")
|
||||
|
||||
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
|
||||
@@ -299,10 +386,11 @@ class ScreenIdentity:
|
||||
|
||||
# Grid items
|
||||
if screen_type == ScreenType.EXPLORE_GRID:
|
||||
actions.append("tap first grid item")
|
||||
actions.append("tap first post")
|
||||
|
||||
# Scroll
|
||||
actions.append("scroll down")
|
||||
actions.append("scroll up")
|
||||
actions.append("press back")
|
||||
|
||||
return list(set(actions)) # Deduplicate
|
||||
|
||||
@@ -117,12 +117,13 @@ class SemanticEvaluator:
|
||||
You are a user with the following interests: {', '.join(persona_interests)}.
|
||||
You are looking at an Instagram post.
|
||||
Evaluate if this post is highly relevant to your interests and if you should like/comment on it.
|
||||
CRITICAL: Check if this post is an advertisement or sponsored content (look for "Sponsored", "Ad", or promotional product placement).
|
||||
|
||||
Reply ONLY in valid JSON format:
|
||||
{{
|
||||
"should_like": true/false,
|
||||
"should_comment": true/false,
|
||||
"reasoning": "brief explanation"
|
||||
"is_ad": true/false
|
||||
}}
|
||||
"""
|
||||
response = self._query_vlm(prompt, screenshot_b64)
|
||||
@@ -131,7 +132,17 @@ class SemanticEvaluator:
|
||||
json_str = response.split("```json")[1].split("```")[0].strip()
|
||||
else:
|
||||
json_str = response.strip()
|
||||
return json.loads(json_str)
|
||||
try:
|
||||
return json.loads(json_str)
|
||||
except json.JSONDecodeError:
|
||||
# Try to close potential unclosed JSON strings
|
||||
if not json_str.endswith("}"):
|
||||
json_str += "}"
|
||||
try:
|
||||
return json.loads(json_str)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
logger.warning(f"👁️ [Vision Core] VLM returned malformed JSON: {response}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to evaluate post vibe: {e}")
|
||||
return None
|
||||
|
||||
@@ -149,10 +149,15 @@ class SpatialParser:
|
||||
# Filter zero-area nodes early
|
||||
if right > left and bottom > top:
|
||||
self._node_counter += 1
|
||||
text_val = attrib.get("text", "").strip()
|
||||
hint_val = attrib.get("hint", "").strip()
|
||||
if not text_val and hint_val:
|
||||
text_val = hint_val
|
||||
|
||||
node = SpatialNode(
|
||||
node_id=f"n_{self._node_counter}",
|
||||
class_name=attrib.get("class", ""),
|
||||
text=attrib.get("text", "").strip(),
|
||||
text=text_val,
|
||||
content_desc=attrib.get("content-desc", "").strip(),
|
||||
resource_id=attrib.get("resource-id", "").strip(),
|
||||
bounds=(left, top, right, bottom),
|
||||
@@ -179,7 +184,7 @@ class SpatialParser:
|
||||
for n in all_nodes:
|
||||
has_semantic = bool(n.text or n.content_desc)
|
||||
semantic_res = n.resource_id and any(
|
||||
x in n.resource_id.lower() for x in ["button", "tab", "icon", "action", "menu"]
|
||||
x in n.resource_id.lower() for x in ["button", "tab", "icon", "action", "menu", "imageview"]
|
||||
)
|
||||
|
||||
if n.clickable or n.scrollable or semantic_res or (has_semantic and n.area < 500000 and n.area > 0):
|
||||
|
||||
@@ -13,7 +13,8 @@ class PersistentList(list):
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
path = f"accounts/{self.filename}.json"
|
||||
base_dir = os.environ.get("GRAMADDICT_ACCOUNTS_DIR", "accounts")
|
||||
path = f"{base_dir}/{self.filename}.json"
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
@@ -27,9 +28,8 @@ class PersistentList(list):
|
||||
self.persist()
|
||||
|
||||
def persist(self, directory=None):
|
||||
if os.environ.get("PYTEST_CURRENT_TEST"):
|
||||
return
|
||||
folder = f"accounts/{directory}" if directory else "accounts"
|
||||
base_dir = os.environ.get("GRAMADDICT_ACCOUNTS_DIR", "accounts")
|
||||
folder = f"{base_dir}/{directory}" if directory else base_dir
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
path = f"{folder}/{self.filename}.json"
|
||||
try:
|
||||
|
||||
@@ -151,16 +151,12 @@ def humanized_scroll(device, is_skip=False, resonance_score=None):
|
||||
|
||||
def humanized_click(device, x, y, double=False, sleep_mod=1.0):
|
||||
"""Simulates a human tap with biomechanical jitter and micro-drift."""
|
||||
body = PhysicsBody.get_session_instance(device)
|
||||
injector = SendEventInjector.get_instance(device)
|
||||
|
||||
def single_tap():
|
||||
points = BezierGesture.tap_curve(x, y, body)
|
||||
# Tap timing: 40-90ms contact time
|
||||
tap_duration = random.uniform(40, 90)
|
||||
timing = BezierGesture.compute_sigmoid_timing(len(points), tap_duration)
|
||||
|
||||
injector.inject_gesture(points, timing, touch_major=body.get_touch_major())
|
||||
# Apply biomechanical jitter
|
||||
jx = int(x + random.gauss(0, 5))
|
||||
jy = int(y + random.gauss(0, 5))
|
||||
device.shell(f"input tap {jx} {jy}")
|
||||
|
||||
if double:
|
||||
# For double tap, the timing is extremely critical (<300ms between taps).
|
||||
|
||||
@@ -18,7 +18,6 @@ correct /dev/input/eventX and the axis ranges on first use.
|
||||
|
||||
import logging
|
||||
import re
|
||||
from time import sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -179,6 +178,9 @@ class SendEventInjector:
|
||||
scale_x = self.x_max / display_w
|
||||
scale_y = self.y_max / display_h
|
||||
|
||||
# Build batch command list
|
||||
cmds = []
|
||||
|
||||
# --- Touch Down (first point) ---
|
||||
x, y, pressure = points[0]
|
||||
ix = int(x * scale_x)
|
||||
@@ -186,8 +188,6 @@ class SendEventInjector:
|
||||
ip = int(pressure * self.pressure_max)
|
||||
itm = min(touch_major, self.touch_major_max)
|
||||
|
||||
# Build batch command for touch-down
|
||||
cmds = []
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_TRACKING_ID} 0")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_X} {ix}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_Y} {iy}")
|
||||
@@ -196,38 +196,36 @@ class SendEventInjector:
|
||||
cmds.append(f"sendevent {dev} {self.EV_KEY} {self.BTN_TOUCH} 1")
|
||||
cmds.append(f"sendevent {dev} {self.EV_SYN} {self.SYN_REPORT} 0")
|
||||
|
||||
# Execute touch-down
|
||||
self.device.shell(" && ".join(cmds))
|
||||
|
||||
# --- Move through intermediate points ---
|
||||
for i in range(1, len(points) - 1):
|
||||
if i - 1 < len(timing_intervals):
|
||||
sleep(timing_intervals[i - 1])
|
||||
delay = timing_intervals[i - 1]
|
||||
if delay > 0.001:
|
||||
cmds.append(f"sleep {delay:.3f}")
|
||||
|
||||
x, y, pressure = points[i]
|
||||
ix = int(x * scale_x)
|
||||
iy = int(y * scale_y)
|
||||
ip = int(pressure * self.pressure_max)
|
||||
|
||||
cmds = []
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_X} {ix}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_Y} {iy}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_PRESSURE} {ip}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_SYN} {self.SYN_REPORT} 0")
|
||||
|
||||
self.device.shell(" && ".join(cmds))
|
||||
|
||||
# --- Touch Up (last point) ---
|
||||
if len(timing_intervals) >= len(points) - 1:
|
||||
sleep(timing_intervals[-1])
|
||||
delay = timing_intervals[-1]
|
||||
else:
|
||||
sleep(0.01)
|
||||
delay = 0.01
|
||||
|
||||
if delay > 0.001:
|
||||
cmds.append(f"sleep {delay:.3f}")
|
||||
|
||||
x, y, pressure = points[-1]
|
||||
ix = int(x * scale_x)
|
||||
iy = int(y * scale_y)
|
||||
|
||||
cmds = []
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_X} {ix}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_POSITION_Y} {iy}")
|
||||
cmds.append(f"sendevent {dev} {self.EV_ABS} {self.ABS_MT_PRESSURE} 0")
|
||||
@@ -235,6 +233,7 @@ class SendEventInjector:
|
||||
cmds.append(f"sendevent {dev} {self.EV_KEY} {self.BTN_TOUCH} 0")
|
||||
cmds.append(f"sendevent {dev} {self.EV_SYN} {self.SYN_REPORT} 0")
|
||||
|
||||
# Execute ALL events in one atomic batch to eliminate ADB latency
|
||||
self.device.shell(" && ".join(cmds))
|
||||
|
||||
except Exception as e:
|
||||
@@ -253,4 +252,12 @@ class SendEventInjector:
|
||||
ex, ey, _ = points[-1]
|
||||
total_ms = int(sum(timing_intervals) * 1000) if timing_intervals else 300
|
||||
|
||||
self.device.shell(f"input swipe {int(sx)} {int(sy)} {int(ex)} {int(ey)} {total_ms}")
|
||||
dist_x = abs(ex - sx)
|
||||
dist_y = abs(ey - sy)
|
||||
|
||||
# Android sometimes interprets a low-duration swipe with minimal movement as a long press or cancels it.
|
||||
# If it's physically a tap (minimal movement, short duration), use native input tap.
|
||||
if dist_x < 15 and dist_y < 15 and total_ms < 150:
|
||||
self.device.shell(f"input tap {int(sx)} {int(sy)}")
|
||||
else:
|
||||
self.device.shell(f"input swipe {int(sx)} {int(sy)} {int(ex)} {int(ey)} {total_ms}")
|
||||
|
||||
@@ -135,64 +135,116 @@ def align_active_post(device):
|
||||
"""
|
||||
aligned = False
|
||||
attempts = 0
|
||||
max_attempts = 3
|
||||
max_attempts = 5 # Increased for structural retry loop
|
||||
failed_bounds = set()
|
||||
|
||||
# Intents for structural discovery
|
||||
intents = [
|
||||
"post author username text (exclude follow buttons)",
|
||||
"post author header profile",
|
||||
"row_feed_photo_profile_name", # ID fallback
|
||||
"clips_viewer_author_container", # Reels fallback
|
||||
"feed post content", # Final desperation
|
||||
]
|
||||
|
||||
while not aligned and attempts < max_attempts:
|
||||
attempts += 1
|
||||
try:
|
||||
xml = device.dump_hierarchy()
|
||||
if "clips_video_container" in xml or "clips_viewer_container" in xml:
|
||||
logger.info("🎯 [Alignment] Reels view detected. Auto-snapping is native.")
|
||||
return True
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
telepath = TelepathicEngine.get_instance()
|
||||
target_node = telepath.find_best_node(xml, "post author header profile", min_confidence=0.4, device=device)
|
||||
|
||||
target_node = None
|
||||
for intent in intents:
|
||||
target_node = telepath.find_best_node(
|
||||
xml, intent, min_confidence=0.35, device=device, track=False, exclude_bounds=list(failed_bounds)
|
||||
)
|
||||
if target_node:
|
||||
break
|
||||
|
||||
if target_node:
|
||||
original_attribs = target_node.get("original_attribs", {})
|
||||
bounds = original_attribs.get("bounds", "")
|
||||
if not bounds:
|
||||
bounds = target_node.get("bounds", "")
|
||||
bounds = original_attribs.get("bounds")
|
||||
|
||||
m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
|
||||
if m:
|
||||
l, t, r, b = map(int, m.groups())
|
||||
header_y = (t + b) // 2
|
||||
|
||||
# Instagram's optimal top margin for a snapped post is ~200-280px
|
||||
target_y = 250
|
||||
diff = header_y - target_y
|
||||
|
||||
# If target is off-center (> 100px), execute precise correction swipe
|
||||
if abs(diff) > 100:
|
||||
info = device.get_info()
|
||||
w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400)
|
||||
cx = w // 2
|
||||
|
||||
max_safe_swipe = int(h * 0.4)
|
||||
|
||||
if diff > 0:
|
||||
# Content is too LOW. Move it UP.
|
||||
dist = min(diff, max_safe_swipe)
|
||||
start_y = int(h * 0.7)
|
||||
end_y = start_y - dist
|
||||
else:
|
||||
# Content is too HIGH. Move it DOWN.
|
||||
dist = min(abs(diff), max_safe_swipe)
|
||||
start_y = int(h * 0.3)
|
||||
end_y = start_y + dist
|
||||
|
||||
# Duration 1.0s = precise mechanical drag with ZERO momentum
|
||||
device.swipe(cx, start_y, cx, end_y, duration=1.0)
|
||||
sleep(1.0)
|
||||
logger.debug(f"📐 [Alignment] Snapping attempt {attempts}: Shifted {diff}px.")
|
||||
bounds_str = ""
|
||||
# If bounds is a tuple from SpatialNode.to_dict()
|
||||
if isinstance(bounds, (tuple, list)) and len(bounds) == 4:
|
||||
left, t, r, b = bounds
|
||||
bounds_str = f"[{left},{t}][{r},{b}]"
|
||||
else:
|
||||
# Fallback to string parsing
|
||||
if not bounds:
|
||||
bounds = target_node.get("bounds", "")
|
||||
m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", str(bounds))
|
||||
if m:
|
||||
left, t, r, b = map(int, m.groups())
|
||||
bounds_str = f"[{left},{t}][{r},{b}]"
|
||||
else:
|
||||
aligned = True
|
||||
logger.warning(f"📐 [Alignment] Could not parse bounds: {bounds}")
|
||||
continue
|
||||
|
||||
# Check if this is a false positive (e.g. bottom bar item misclassified)
|
||||
# Post headers should be in the top half usually, or at least not at the very bottom
|
||||
info = device.get_info()
|
||||
h = info.get("displayHeight", 2400)
|
||||
if t > h * 0.85:
|
||||
logger.debug(f"📐 [Alignment] Rejecting node at y={t} (too low, likely bottom bar)")
|
||||
failed_bounds.add(bounds_str)
|
||||
continue
|
||||
|
||||
header_y = (t + b) // 2
|
||||
target_y = 250 # Top margin for headers
|
||||
diff = header_y - target_y
|
||||
|
||||
# If target is off-center (> 50px for higher precision), execute precise correction swipe
|
||||
if abs(diff) > 50:
|
||||
info = device.get_info()
|
||||
w = info.get("displayWidth", 1080)
|
||||
cx = w // 2
|
||||
|
||||
max_safe_swipe = int(h * 0.4)
|
||||
|
||||
# Calculate movement
|
||||
dist = min(abs(diff), max_safe_swipe)
|
||||
if diff > 0:
|
||||
# Content is too LOW. Move it UP (Swipe UP).
|
||||
start_y = int(h * 0.7)
|
||||
end_y = start_y - dist
|
||||
else:
|
||||
# Content is too HIGH. Move it DOWN (Swipe DOWN).
|
||||
start_y = int(h * 0.3)
|
||||
end_y = start_y + dist
|
||||
|
||||
logger.debug(f"📐 [Alignment] Attempt {attempts}: Snapping {diff}px (Swipe {start_y} -> {end_y})")
|
||||
# Duration 1.5s = ultra-precise mechanical drag with ZERO momentum
|
||||
device.swipe(cx, start_y, cx, end_y, duration=1.5)
|
||||
sleep(1.0)
|
||||
|
||||
# Refresh XML for next iteration check
|
||||
continue
|
||||
else:
|
||||
logger.info(f"🎯 [Alignment] Perfect snap achieved after {attempts} attempts.")
|
||||
aligned = True
|
||||
else:
|
||||
break # No header found, cannot align
|
||||
logger.debug(f"📐 [Alignment] No structural markers found on attempt {attempts}.")
|
||||
# If we can't find any markers, maybe we are stuck in a transition.
|
||||
# Micro-wobble to force a layout update.
|
||||
if attempts < 3:
|
||||
info = device.get_info()
|
||||
w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400)
|
||||
device.swipe(w // 2, h // 2, w // 2, h // 2 - 20, duration=0.2)
|
||||
sleep(0.5)
|
||||
device.swipe(w // 2, h // 2 - 20, w // 2, h // 2, duration=0.2)
|
||||
sleep(1.0)
|
||||
else:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"📐 [Alignment] Snapping correction failed: {e}")
|
||||
break
|
||||
|
||||
if aligned and attempts > 1:
|
||||
logger.debug(f"📐 [Alignment] Snapped post cleanly into view after {attempts} attempts.")
|
||||
return True
|
||||
return aligned
|
||||
|
||||
@@ -121,31 +121,11 @@ class QNavGraph:
|
||||
GOAP-powered action execution.
|
||||
Replaces _execute_transition() for post interactions.
|
||||
|
||||
Screen-aware: refuses to attempt actions that don't exist on the current screen.
|
||||
|
||||
Usage:
|
||||
nav_graph.do("like this post") # instead of _execute_transition("tap_like_button")
|
||||
nav_graph.do("follow this user") # instead of _execute_transition("tap_follow_button")
|
||||
nav_graph.do("tap first grid item") # instead of _execute_transition("tap_explore_grid_item")
|
||||
"""
|
||||
# ── Screen sanity check: is this action possible here? ──
|
||||
screen = self.goap.perceive()
|
||||
available = screen.get("available_actions", [])
|
||||
screen_type = screen["screen_type"]
|
||||
|
||||
# Map goal to the action that should be available
|
||||
action_checks = {
|
||||
"like": "tap like button",
|
||||
"comment": "tap comment button",
|
||||
"share": "tap share button",
|
||||
}
|
||||
for keyword, required_action in action_checks.items():
|
||||
if keyword in goal.lower() and required_action not in available:
|
||||
logger.warning(
|
||||
f"🚫 [GOAP] Cannot '{goal}' on {screen_type.value} "
|
||||
f"('{required_action}' not available on this screen)"
|
||||
)
|
||||
return False
|
||||
|
||||
return self.goap._execute_action(goal)
|
||||
|
||||
@@ -171,13 +151,13 @@ class QNavGraph:
|
||||
success = self.sae.ensure_clear_screen(max_attempts=max_attempts + 5, initial_xml=xml_dump)
|
||||
return success
|
||||
|
||||
def _execute_transition(self, action: str, mock_semantic_engine=None, max_retries: int = 2) -> bool:
|
||||
def _execute_transition(self, action: str, max_retries: int = 2) -> bool:
|
||||
"""
|
||||
Executes a transition (e.g. 'tap_explore_tab') using the Telepathic Semantic Engine.
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
engine = mock_semantic_engine or TelepathicEngine.get_instance()
|
||||
engine = TelepathicEngine.get_instance()
|
||||
|
||||
failed_positions = set() # Track (x, y) of clicks that failed, for grid retry diversity
|
||||
|
||||
@@ -209,7 +189,7 @@ class QNavGraph:
|
||||
# Grid & Profile
|
||||
"tap_explore_grid_item": "first image in explore grid",
|
||||
"tap_story_tray_item": "profile picture avatar story ring",
|
||||
"tap_follow_button": "tap follow button on profile",
|
||||
"tap_follow_button": "tap 'Follow' button on profile",
|
||||
"tap_grid_first_post": "first image post in profile grid",
|
||||
"tap_back": "tap back button icon arrow",
|
||||
"tap_message_icon": "tap direct message icon inbox",
|
||||
|
||||
@@ -125,10 +125,10 @@ class QdrantBase:
|
||||
if key:
|
||||
headers["Authorization"] = f"Bearer {key}"
|
||||
# OpenAI/OpenRouter use 'input' instead of 'prompt'
|
||||
payload = {"model": model, "input": str(text)[:8000]}
|
||||
payload = {"model": model, "input": str(text)[:2000]}
|
||||
else:
|
||||
# Local Ollama
|
||||
payload = {"model": model, "prompt": str(text)[:8000]}
|
||||
payload = {"model": model, "prompt": str(text)[:2000]}
|
||||
|
||||
# Log to prevent user from thinking the bot is hung during model swap in VRAM
|
||||
if not getattr(self, "_has_logged_embedding", False):
|
||||
@@ -141,7 +141,7 @@ class QdrantBase:
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=12,
|
||||
timeout=30,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.debug(f"Embedding API Error {resp.status_code}: {resp.text}")
|
||||
@@ -155,8 +155,8 @@ class QdrantBase:
|
||||
return data["data"][0]["embedding"]
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to generate embedding via {url}: {e}")
|
||||
return None
|
||||
logger.error(f"Failed to generate embedding via {url}: {e}")
|
||||
raise
|
||||
|
||||
def generate_uuid(self, seed_string: str) -> str:
|
||||
"""
|
||||
@@ -184,6 +184,7 @@ class QdrantBase:
|
||||
self.client.upsert(
|
||||
collection_name=self.collection_name,
|
||||
points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)],
|
||||
wait=True,
|
||||
)
|
||||
|
||||
# ABSOLUTE LOGGING: User requirement for full observability
|
||||
@@ -361,7 +362,7 @@ class UIMemoryDB(QdrantBase):
|
||||
sig = re.sub(r"\s+", " ", sig).strip()
|
||||
|
||||
# 3. Strict truncation for nomic-embed-text context window
|
||||
return sig[:4000]
|
||||
return sig[:2000]
|
||||
|
||||
def _deterministic_id(self, intent: str) -> str:
|
||||
"""
|
||||
@@ -429,8 +430,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if exact_points:
|
||||
eval_result = _evaluate_payload(exact_points[0].payload, score=1.0, point_id=point_id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via EXACT ID MATCH! (Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
# If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
|
||||
@@ -459,8 +461,9 @@ class UIMemoryDB(QdrantBase):
|
||||
if results and results[0].score >= similarity_threshold:
|
||||
eval_result = _evaluate_payload(results[0].payload, score=results[0].score, point_id=results[0].id)
|
||||
if eval_result:
|
||||
logger.debug(
|
||||
f"Resolved intent '{intent}' from Qdrant Memory via vector search! (Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})"
|
||||
logger.info(
|
||||
f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
|
||||
extra={"color": "\x1b[36m"}, # Cyan color
|
||||
)
|
||||
return eval_result["solution"]
|
||||
return None
|
||||
@@ -511,7 +514,10 @@ class UIMemoryDB(QdrantBase):
|
||||
],
|
||||
wait=True,
|
||||
)
|
||||
logger.info(f"Learned pattern for '{intent}' and saved to Qdrant Memory (ID: {point_id[:8]}...).")
|
||||
logger.info(
|
||||
f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
|
||||
extra={"color": "\x1b[35m"}, # Magenta color
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Qdrant storage error: {e}")
|
||||
|
||||
@@ -573,7 +579,12 @@ class UIMemoryDB(QdrantBase):
|
||||
payload={"confidence": new_confidence},
|
||||
points=[point_id],
|
||||
)
|
||||
logger.debug(f"Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f}).")
|
||||
color = "\x1b[32m" if delta > 0 else "\x1b[31m" # Green for positive, Red for negative
|
||||
symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
|
||||
logger.info(
|
||||
f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
|
||||
extra={"color": color},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Confidence adjustment error: {e}")
|
||||
|
||||
@@ -1188,6 +1199,25 @@ class ParasocialCRMDB(QdrantBase):
|
||||
log_success=f"🧠 [ParasocialCRM] Updated @{username} into Qdrant. Stage {stage} ({intent_type})",
|
||||
)
|
||||
|
||||
def enrich_lead(self, username: str, data: dict):
|
||||
"""
|
||||
Enriches a lead with scraped data.
|
||||
"""
|
||||
if not self.is_connected:
|
||||
return
|
||||
|
||||
current = self.get_relationship_stage(username)
|
||||
current.update(data)
|
||||
|
||||
vector = self._get_embedding(f"User: {username}")
|
||||
if vector:
|
||||
self.upsert_point(
|
||||
seed_string=f"User_{username}",
|
||||
vector=vector,
|
||||
payload=current,
|
||||
log_success=f"🧠 [ParasocialCRM] Enriched @{username} data.",
|
||||
)
|
||||
|
||||
def log_generated_comment(self, username: str, comment_text: str):
|
||||
"""Phase 10: RAG memory point for specific users."""
|
||||
if not self.is_connected:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import math
|
||||
import random
|
||||
from typing import Optional
|
||||
|
||||
from colorama import Fore
|
||||
@@ -331,14 +332,32 @@ class ResonanceEngine:
|
||||
is_comment_node = "comment" in res_id or "textview" in res_id
|
||||
|
||||
# 3. Block accessibility garbage & UI labels
|
||||
# Zero-Maintenance: Only structural patterns. Short strings
|
||||
# (< 5 chars) from UI buttons are blocked by length, not by
|
||||
# translating every possible language.
|
||||
is_ui_junk = (
|
||||
val.lower().startswith("go to")
|
||||
or val.lower().startswith("tap to")
|
||||
or "actions for this post" in val.lower()
|
||||
or len(val.strip()) < 3
|
||||
)
|
||||
|
||||
# Block known English UI action labels.
|
||||
# We intentionally do NOT add German/Spanish/etc translations.
|
||||
# Instead, we rely on the structural `is_comment_node` filter
|
||||
# above + length heuristic to catch non-comment UI elements.
|
||||
blocked_exact = [
|
||||
"reply",
|
||||
"like",
|
||||
"view replies",
|
||||
"see translation",
|
||||
"hide replies",
|
||||
"view all comments",
|
||||
"send",
|
||||
]
|
||||
|
||||
if val and len(val) > 2 and is_comment_node and not is_ui_junk:
|
||||
if val.lower() not in ["reply", "like", "view replies", "see translation", "hide replies"]:
|
||||
if val.lower() not in blocked_exact:
|
||||
raw_comments.append(val)
|
||||
except Exception as e:
|
||||
logger.error(f"🧠 [Comment Learning] Failed to parse XML: {e}")
|
||||
@@ -393,7 +412,7 @@ class ResonanceEngine:
|
||||
logger.debug(f"DEBUG CONDENSER RAW: {response_text}")
|
||||
|
||||
# Parse json gracefully
|
||||
if type(response_text) is str:
|
||||
if isinstance(response_text, str):
|
||||
clean_json = response_text.strip()
|
||||
if clean_json.startswith("```json"):
|
||||
clean_json = clean_json[7:]
|
||||
|
||||
@@ -33,6 +33,7 @@ class ScreenTopology:
|
||||
"tap profile tab": ScreenType.OWN_PROFILE,
|
||||
"tap reels tab": ScreenType.REELS_FEED,
|
||||
"tap messages tab": ScreenType.DM_INBOX,
|
||||
"tap story ring avatar": ScreenType.STORY_VIEW,
|
||||
},
|
||||
ScreenType.EXPLORE_GRID: {
|
||||
"tap home tab": ScreenType.HOME_FEED,
|
||||
@@ -57,9 +58,17 @@ class ScreenTopology:
|
||||
ScreenType.FOLLOW_LIST: {
|
||||
"press back": ScreenType.OWN_PROFILE,
|
||||
},
|
||||
ScreenType.STORY_VIEW: {
|
||||
"press back": ScreenType.HOME_FEED,
|
||||
},
|
||||
ScreenType.OTHER_PROFILE: {
|
||||
"press back": ScreenType.HOME_FEED,
|
||||
},
|
||||
ScreenType.POST_DETAIL: {
|
||||
"tap home tab": ScreenType.HOME_FEED,
|
||||
"tap explore tab": ScreenType.EXPLORE_GRID,
|
||||
"tap profile tab": ScreenType.OWN_PROFILE,
|
||||
"tap reels tab": ScreenType.REELS_FEED,
|
||||
},
|
||||
ScreenType.UNKNOWN: {
|
||||
"tap home tab": ScreenType.HOME_FEED,
|
||||
@@ -88,7 +97,9 @@ class ScreenTopology:
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def find_route(cls, from_screen: ScreenType, to_screen: ScreenType) -> Optional[List[Tuple[str, ScreenType]]]:
|
||||
def find_route(
|
||||
cls, from_screen: ScreenType, to_screen: ScreenType, avoid_actions: set = None
|
||||
) -> Optional[List[Tuple[str, ScreenType]]]:
|
||||
"""
|
||||
BFS shortest path from from_screen to to_screen.
|
||||
|
||||
@@ -100,6 +111,8 @@ class ScreenTopology:
|
||||
if from_screen == to_screen:
|
||||
return []
|
||||
|
||||
avoid_actions = avoid_actions or set()
|
||||
|
||||
queue: deque = deque()
|
||||
queue.append((from_screen, []))
|
||||
visited = {from_screen}
|
||||
@@ -109,6 +122,9 @@ class ScreenTopology:
|
||||
transitions = cls.TRANSITIONS.get(current, {})
|
||||
|
||||
for action, next_screen in transitions.items():
|
||||
if action in avoid_actions or action.replace(" ", "_") in avoid_actions:
|
||||
continue
|
||||
|
||||
if next_screen == to_screen:
|
||||
return path + [(action, next_screen)]
|
||||
|
||||
|
||||
@@ -277,7 +277,29 @@ class SessionState:
|
||||
|
||||
|
||||
class SessionStateEncoder(JSONEncoder):
|
||||
"""JSON encoder for SessionState that is crash-proof against non-serializable types."""
|
||||
|
||||
_SAFE_TYPES = (str, int, float, bool, type(None))
|
||||
|
||||
@classmethod
|
||||
def _sanitize_value(cls, value):
|
||||
"""Convert any non-JSON-serializable value to a safe string representation."""
|
||||
if isinstance(value, cls._SAFE_TYPES):
|
||||
return value
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, dict):
|
||||
return {k: cls._sanitize_value(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [cls._sanitize_value(v) for v in value]
|
||||
# Last resort: stringify unknown objects to prevent json.dump mid-write crashes
|
||||
return str(value)
|
||||
|
||||
def default(self, session_state: SessionState):
|
||||
# Sanitize args dict — never trust raw __dict__, it may contain datetime or other garbage
|
||||
raw_args = session_state.args.__dict__ if hasattr(session_state.args, "__dict__") else {}
|
||||
safe_args = {k: self._sanitize_value(v) for k, v in raw_args.items()}
|
||||
|
||||
return {
|
||||
"id": session_state.id,
|
||||
"total_interactions": sum(session_state.totalInteractions.values()),
|
||||
@@ -291,7 +313,7 @@ class SessionStateEncoder(JSONEncoder):
|
||||
"total_scraped": session_state.totalScraped,
|
||||
"start_time": str(session_state.startTime),
|
||||
"finish_time": str(session_state.finishTime),
|
||||
"args": session_state.args.__dict__,
|
||||
"args": safe_args,
|
||||
"profile": {
|
||||
"posts": session_state.my_posts_count,
|
||||
"followers": session_state.my_followers_count,
|
||||
|
||||
@@ -270,7 +270,13 @@ class SituationalAwarenessEngine:
|
||||
if clickable == "true":
|
||||
parts.append("CLICKABLE")
|
||||
if bounds:
|
||||
parts.append(f"bounds={bounds}")
|
||||
nums = [int(n) for n in re.findall(r"\d+", bounds)]
|
||||
if len(nums) == 4:
|
||||
cx = (nums[0] + nums[2]) // 2
|
||||
cy = (nums[1] + nums[3]) // 2
|
||||
parts.append(f"bounds={bounds} center=({cx},{cy})")
|
||||
else:
|
||||
parts.append(f"bounds={bounds}")
|
||||
|
||||
elements.append(" | ".join(parts))
|
||||
|
||||
@@ -293,8 +299,6 @@ class SituationalAwarenessEngine:
|
||||
if not xml_dump or not isinstance(xml_dump, str):
|
||||
return SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
xml_dump.lower()
|
||||
|
||||
blocked_markers = [
|
||||
"try again later",
|
||||
"action blocked",
|
||||
@@ -346,8 +350,31 @@ class SituationalAwarenessEngine:
|
||||
is_foreign = True
|
||||
|
||||
if is_foreign:
|
||||
# We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks
|
||||
# for Android System UI variations across different device manufacturers.
|
||||
# ── Tier 1: Known Foreign Packages (O(1) — ZERO LLM) ──
|
||||
# Production bug 2026-05-03: Play Store was detected via slow LLM path.
|
||||
# For these well-known packages, a set lookup is instant and infallible.
|
||||
KNOWN_FOREIGN_PACKAGES = {
|
||||
"com.android.vending", # Play Store
|
||||
"com.android.chrome", # Chrome
|
||||
"com.google.android.chrome", # Chrome (Google build)
|
||||
"com.google.android.youtube", # YouTube
|
||||
"org.mozilla.firefox", # Firefox
|
||||
"com.opera.browser", # Opera
|
||||
"com.brave.browser", # Brave
|
||||
"com.microsoft.emmx", # Edge
|
||||
"com.sec.android.app.sbrowser", # Samsung Browser
|
||||
}
|
||||
dominant_pkgs = packages - {"com.android.systemui"}
|
||||
fast_match = dominant_pkgs & KNOWN_FOREIGN_PACKAGES
|
||||
if fast_match:
|
||||
logger.info(
|
||||
f"🚨 [SAE Perceive] Known foreign package: {fast_match} → "
|
||||
f"OBSTACLE_FOREIGN_APP (O(1) fast-path, no LLM needed)"
|
||||
)
|
||||
return SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
# ── Tier 2: Unknown/Ambiguous Packages → LLM Classification ──
|
||||
# Only SystemUI-only or rare custom packages reach this path.
|
||||
try:
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
@@ -369,8 +396,8 @@ class SituationalAwarenessEngine:
|
||||
args = Config().args
|
||||
except Exception:
|
||||
pass
|
||||
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_model", "qwen3.5:latest")
|
||||
url = getattr(args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
|
||||
res = query_telepathic_llm(
|
||||
model=model,
|
||||
@@ -406,25 +433,6 @@ class SituationalAwarenessEngine:
|
||||
|
||||
compressed = self._compress_xml(xml_dump)
|
||||
|
||||
# ── Structural Fast-Check: Content-Creation Overlays ──
|
||||
# These full-screen overlays live INSIDE Instagram's package but block
|
||||
# all normal navigation. They are invisible to the foreign-app detector
|
||||
# and frequently fool the LLM into thinking they are "normal" browsing.
|
||||
# Detecting them structurally is O(1) and requires ZERO LLM calls.
|
||||
creation_flow_markers = (
|
||||
"quick_capture", # Camera / story capture overlay
|
||||
"gallery_cancel_button", # Story gallery "Back to Home" button
|
||||
"creation_flow", # Post creation wizard
|
||||
"reel_camera", # Reel recording interface
|
||||
)
|
||||
|
||||
# Guard: Check against compressed string to ensure these markers ONLY appear
|
||||
# as resource IDs (e.g. "id=quick_capture_...") and not as plain text in
|
||||
# user comments/bios (which would look like "text='... creation_flow ...'")
|
||||
if any(re.search(rf"id=[^\s|]*{marker}", compressed, re.IGNORECASE) for marker in creation_flow_markers):
|
||||
logger.info("🧠 [SAE Perceive] Content-creation overlay detected structurally → OBSTACLE_MODAL")
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
cached_type = screen_memory.get_screen_type(compressed)
|
||||
|
||||
if cached_type:
|
||||
@@ -433,6 +441,80 @@ class SituationalAwarenessEngine:
|
||||
elif cached_type == "NORMAL":
|
||||
return SituationType.NORMAL
|
||||
|
||||
# ── Structural Fast-Check: Content-Creation Overlays ──
|
||||
# These full-screen overlays live INSIDE Instagram's package but block
|
||||
# all normal navigation. They are invisible to the foreign-app detector
|
||||
# and frequently fool the LLM into thinking they are "normal" browsing.
|
||||
# Detecting them structurally is O(1) and requires ZERO LLM calls.
|
||||
# This is checked AFTER Qdrant to ensure that if the LLM unlearned a false positive,
|
||||
# we respect the learned NORMAL state and don't infinite-loop.
|
||||
creation_flow_markers = (
|
||||
"quick_capture", # Camera / story capture overlay
|
||||
"gallery_cancel_button", # Story gallery "Back to Home" button
|
||||
"creation_flow", # Post creation wizard
|
||||
"reel_camera", # Reel recording interface
|
||||
)
|
||||
|
||||
# Guard: Use the RAW xml_dump to avoid truncation of root containers (Z-index filtering),
|
||||
# but ensure we only match inside resource-id attributes to prevent false positives from user text.
|
||||
if any(
|
||||
re.search(rf'resource-id="[^"]*{marker}[^"]*"', xml_dump, re.IGNORECASE) for marker in creation_flow_markers
|
||||
):
|
||||
logger.info("🧠 [SAE Perceive] Content-creation overlay detected structurally → OBSTACLE_MODAL")
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
|
||||
# ── Structural Fast-Check: Instagram-Internal Modal Overlays ──
|
||||
# Surveys, rating prompts, and interstitial modals live INSIDE Instagram's
|
||||
# package but block normal interaction. They share a common structural
|
||||
# pattern: a container resource-id containing "survey", "interstitial",
|
||||
# or "nux_" (new-user-experience), plus dismiss buttons ("Not Now").
|
||||
# Detecting them structurally is O(1) and eliminates LLM hallucination risk.
|
||||
instagram_modal_markers = (
|
||||
"survey_overlay_container", # "How are you enjoying Instagram?" survey
|
||||
"survey_title", # Survey title text view
|
||||
"interstitial_container", # Generic interstitial blocker
|
||||
"mystery_interstitial", # Unknown/dynamic interstitials
|
||||
"nux_overlay", # New-user-experience onboarding modals
|
||||
"rating_prompt", # App Store rating prompt
|
||||
"feedback_dialog", # Feedback collection dialogs
|
||||
)
|
||||
if any(
|
||||
re.search(rf'resource-id="[^"]*{marker}[^"]*"', xml_dump, re.IGNORECASE)
|
||||
for marker in instagram_modal_markers
|
||||
):
|
||||
logger.info("🧠 [SAE Perceive] Instagram modal overlay detected structurally → OBSTACLE_MODAL")
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
|
||||
# Fallback heuristic: detect modals by dismiss-button text patterns.
|
||||
# If we see "Not Now" or "Take Survey" as button text inside Instagram, it's a modal.
|
||||
# Guard: match ONLY inside short text attributes (< 40 chars) to avoid caption false positives.
|
||||
dismiss_button_patterns = (
|
||||
r'text="Not Now"',
|
||||
r'text="not now"',
|
||||
r'text="Nicht jetzt"', # German: "Not Now"
|
||||
r'text="Take Survey"',
|
||||
r'text="rate \d+ stars?"', # "rate 5 stars"
|
||||
r'text="Bewerten"', # German: "Rate"
|
||||
)
|
||||
has_dismiss_button = any(re.search(p, xml_dump, re.IGNORECASE) for p in dismiss_button_patterns)
|
||||
if has_dismiss_button:
|
||||
# Cross-validate: must also have a container that looks like a dialog/overlay
|
||||
# (not just a random "Not Now" text in a DM thread or post caption)
|
||||
has_overlay_structure = bool(
|
||||
re.search(
|
||||
r'resource-id="[^"]*(?:overlay|dialog|interstitial|survey|sheet|prompt)[^"]*"',
|
||||
xml_dump,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
or re.search(r'resource-id="[^"]*button_(?:negative|positive)[^"]*"', xml_dump, re.IGNORECASE)
|
||||
)
|
||||
if has_overlay_structure:
|
||||
logger.info("🧠 [SAE Perceive] Instagram dismiss-button modal detected structurally → OBSTACLE_MODAL")
|
||||
screen_memory.store_screen(compressed, "OBSTACLE_MODAL")
|
||||
return SituationType.OBSTACLE_MODAL
|
||||
|
||||
# If not cached, query LLM for autonomous structural classification
|
||||
try:
|
||||
from GramAddict.core.config import Config
|
||||
@@ -440,7 +522,7 @@ class SituationalAwarenessEngine:
|
||||
|
||||
prompt = (
|
||||
"You are a Situation Classifier for a mobile automation agent.\n"
|
||||
"Analyze the given Android UI XML dump. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"Analyze the given Android UI XML dump AND screenshot. Is there a blocking MODAL, DIALOG, or POPUP "
|
||||
"covering the screen that needs to be dismissed, or is this a NORMAL usable screen?\n"
|
||||
"A 'clean_sheet_container' with standard Instagram feed content is NORMAL.\n"
|
||||
"A survey, rating prompt, 'not now' prompt, or permission dialog is an OBSTACLE_MODAL.\n"
|
||||
@@ -457,11 +539,17 @@ class SituationalAwarenessEngine:
|
||||
args = Config().args
|
||||
except Exception:
|
||||
pass
|
||||
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
res = query_telepathic_llm(
|
||||
model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True
|
||||
model=model,
|
||||
url=url,
|
||||
system_prompt="Strict JSON classifier.",
|
||||
user_prompt=prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
use_local_edge=True,
|
||||
)
|
||||
import json
|
||||
|
||||
@@ -502,27 +590,31 @@ class SituationalAwarenessEngine:
|
||||
Called ONLY when recall AND structural planning both miss.
|
||||
"""
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
try:
|
||||
args = Config().args
|
||||
model = getattr(args, "ai_fallback_model", "llama3.2:1b")
|
||||
url = getattr(args, "ai_fallback_url", "http://localhost:11434/api/generate")
|
||||
model = getattr(args, "ai_telepathic_model", "llava:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
except Exception:
|
||||
model = "llama3.2:1b"
|
||||
model = "llava:latest"
|
||||
url = "http://localhost:11434/api/generate"
|
||||
|
||||
system_prompt = (
|
||||
"You are an Android UI navigation agent. Your job is to escape obstacles "
|
||||
"(dialogs, modals, foreign apps, system popups) and return to Instagram. "
|
||||
"Analyze the screen content and return a JSON escape action.\n\n"
|
||||
"Analyze the screen content (Screenshot AND XML) and return a JSON escape action.\n\n"
|
||||
"Rules:\n"
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is OBSTACLE_LOCKED_SCREEN, action must be 'unlock'\n"
|
||||
"- If the Situation type is OBSTACLE_FOREIGN_APP, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_locked_screen, action must be 'unlock'\n"
|
||||
"- If the Situation type is obstacle_foreign_app, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_system, you MUST look for 'Deny', 'Don't allow', or 'Cancel' and click it. \n"
|
||||
" NEVER click 'Allow', 'OK', or 'Confirm' on system permissions.\n"
|
||||
" If no negative action button exists, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- If nothing else works, suggest 'app_start' to force-reopen Instagram\n"
|
||||
"- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n"
|
||||
"- When you choose to click, you MUST use the EXACT coordinates provided in `center=(x,y)` for that element in the XML\n"
|
||||
'- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}'
|
||||
)
|
||||
|
||||
@@ -533,20 +625,31 @@ class SituationalAwarenessEngine:
|
||||
user_prompt += "What action should I take to clear this obstacle and return to Instagram? Return JSON only."
|
||||
|
||||
try:
|
||||
resp = query_llm(
|
||||
screenshot_b64 = getattr(self.device, "get_screenshot_b64", lambda: None)()
|
||||
|
||||
resp = query_telepathic_llm(
|
||||
url=url,
|
||||
model=model,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=True,
|
||||
timeout=30,
|
||||
max_tokens=300,
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
images_b64=[screenshot_b64] if screenshot_b64 else None,
|
||||
temperature=0.0,
|
||||
)
|
||||
if resp and "response" in resp:
|
||||
if resp:
|
||||
import json
|
||||
|
||||
data = json.loads(resp["response"])
|
||||
try:
|
||||
data = json.loads(resp)
|
||||
except json.JSONDecodeError:
|
||||
# Try extracting JSON via regex if LLM was chatty
|
||||
import re
|
||||
|
||||
match = re.search(r"\{.*\}", resp, re.DOTALL)
|
||||
if match:
|
||||
data = json.loads(match.group(0))
|
||||
else:
|
||||
raise ValueError(f"Could not parse JSON from: {resp}")
|
||||
|
||||
return EscapeAction(
|
||||
action_type=data.get("action", "back"),
|
||||
x=int(data.get("x", 0)),
|
||||
@@ -658,6 +761,24 @@ class SituationalAwarenessEngine:
|
||||
|
||||
logger.warning(f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})")
|
||||
|
||||
# ── O(1) Fast-Path for Foreign Apps ──
|
||||
if situation == SituationType.OBSTACLE_FOREIGN_APP:
|
||||
logger.warning("⚡ [SAE Fast-Path] Foreign App detected. Bypassing LLM and killing immediately.")
|
||||
action = EscapeAction("kill_foreign_apps", reason="O(1) fast-path to eliminate foreign app")
|
||||
self._execute_escape(action)
|
||||
|
||||
# Check if we recovered
|
||||
post_xml = self.device.dump_hierarchy()
|
||||
if self.perceive(post_xml) == SituationType.NORMAL:
|
||||
logger.info("✅ [SAE Fast-Path] Foreign App cleared successfully!")
|
||||
self._consecutive_failures = 0
|
||||
return True
|
||||
|
||||
# If we didn't recover, log it and let the loop continue
|
||||
logger.warning("⚠️ [SAE Fast-Path] kill_foreign_apps did not return to NORMAL. Retrying...")
|
||||
self._consecutive_failures += 1
|
||||
continue
|
||||
|
||||
# ── COMPRESS for memory lookup ──
|
||||
compressed = self._compress_xml(xml_dump)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from time import sleep
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ghost_type(device, text: str):
|
||||
def ghost_type(device, text: str, speed: str = "normal"):
|
||||
"""
|
||||
Tesla Stealth Ghost Keyboard.
|
||||
Bypasses UIAutomator virtual IME completely and sends raw Native InputEvents.
|
||||
@@ -48,6 +48,10 @@ def ghost_type(device, text: str):
|
||||
else:
|
||||
_adb_inject_text(device, chunk)
|
||||
|
||||
if speed == "fast":
|
||||
sleep(random.uniform(0.01, 0.05))
|
||||
continue
|
||||
|
||||
# Realistic pause between semantic bursts (humans think while typing)
|
||||
if chunk.endswith((" ", ".", ",", "!", "?")):
|
||||
sleep(random.uniform(0.2, 0.5))
|
||||
|
||||
@@ -53,34 +53,21 @@ class TelepathicEngine:
|
||||
# Core Resolution Engine
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def find_best_node(self, xml_string: str, intent_description: str, device=None, **kwargs) -> Optional[dict]:
|
||||
def find_best_node(
|
||||
self,
|
||||
xml_string: str,
|
||||
intent_description: str,
|
||||
device=None,
|
||||
track: bool = True,
|
||||
exclude_bounds: list[str] = None,
|
||||
**kwargs,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Public facade for resolving a node.
|
||||
Translates Android UI bounds into standard GramAddict node dicts.
|
||||
"""
|
||||
logger.debug(f"🧠 [SpatialEngine] Resolving intent: '{intent_description}'")
|
||||
|
||||
# 0. DM Thread Guard: Block profile intents inside DM threads
|
||||
is_dm_thread = "direct_thread_header" in xml_string or "row_thread_composer_edittext" in xml_string
|
||||
if is_dm_thread:
|
||||
profile_keywords = ["profile", "follow", "first image", "grid", "avatar", "story ring", "feed"]
|
||||
if any(k in intent_description.lower() for k in profile_keywords):
|
||||
logger.warning(f"🛡️ [DM Guard] Blocked profile/feed intent '{intent_description}' inside DM thread.")
|
||||
return {"blocked_by_dm_thread": True}
|
||||
|
||||
# 0.5 Comments Disabled Guard
|
||||
if "comment" in intent_description.lower():
|
||||
if "comments are turned off" in xml_string.lower():
|
||||
logger.warning("🛡️ [Comment Guard] Comments are disabled on this post.")
|
||||
return {"skip": True, "semantic": "comments disabled"}
|
||||
|
||||
# 1.25 Grid Fast-Path (Deterministically bypass VLM for first grid item)
|
||||
if "first image in explore grid" in intent_description.lower():
|
||||
nodes_dicts = self._extract_semantic_nodes(xml_string)
|
||||
fast_node = self._grid_fast_path(intent_description, nodes_dicts, kwargs.get("skip_positions"))
|
||||
if fast_node:
|
||||
return fast_node
|
||||
|
||||
# 1. Parse into Spatial Topology
|
||||
root = self._parser.parse(xml_string)
|
||||
if not root:
|
||||
@@ -90,24 +77,48 @@ class TelepathicEngine:
|
||||
# 2. Extract interactable candidates
|
||||
candidates = self._parser.get_clickable_nodes(root)
|
||||
|
||||
if exclude_bounds:
|
||||
filtered_candidates = []
|
||||
for c in candidates:
|
||||
bounds_str = f"[{c.x1},{c.y1}][{c.x2},{c.y2}]"
|
||||
if bounds_str not in exclude_bounds:
|
||||
filtered_candidates.append(c)
|
||||
candidates = filtered_candidates
|
||||
|
||||
# 3. Resolve intent against candidates
|
||||
best_node = self._resolver.resolve(intent_description, candidates)
|
||||
best_node = self._resolver.resolve(intent_description, candidates, device=device)
|
||||
|
||||
if not best_node:
|
||||
logger.warning(f"No viable nodes found for intent: '{intent_description}'")
|
||||
return None
|
||||
|
||||
# 3.1 BUG 7 Fix: Semantic Guard for 'post media content'
|
||||
intent_lower = intent_description.lower()
|
||||
semantic_str = (
|
||||
(best_node.text or "") + " " + (best_node.content_desc or "") + " " + (best_node.resource_id or "")
|
||||
).lower()
|
||||
if "post media content" in intent_lower:
|
||||
if "follow" in semantic_str.replace("_", " "):
|
||||
logger.warning("🚫 [SpatialEngine] VLM selected a 'Follow' button for 'post media content'. Blocked.")
|
||||
return None
|
||||
|
||||
# 3.5 Following Button Guard
|
||||
if "follow" in intent_description.lower() and "unfollow" not in intent_description.lower():
|
||||
if (
|
||||
"follow" in intent_description.lower()
|
||||
and "unfollow" not in intent_description.lower()
|
||||
and "following" not in intent_description.lower()
|
||||
):
|
||||
semantic = (
|
||||
(best_node.text or "") + " " + (best_node.content_desc or "") + " " + (best_node.resource_id or "")
|
||||
)
|
||||
semantic = semantic.lower()
|
||||
if "following" in semantic or "gefolgt" in semantic or "requested" in semantic or "angefragt" in semantic:
|
||||
# Zero-Maintenance: Only English UI states. resource_id never changes with locale.
|
||||
if "following" in semantic or "requested" in semantic:
|
||||
return {"skip": True, "semantic": "already_followed"}
|
||||
|
||||
# 4. Track action
|
||||
self._memory.track_click(intent_description, best_node, xml_string)
|
||||
if track:
|
||||
self._memory.track_click(intent_description, best_node, xml_string)
|
||||
|
||||
# Translate to old GramAddict dict format for backward compatibility
|
||||
return self._translate_node(best_node)
|
||||
@@ -144,27 +155,6 @@ class TelepathicEngine:
|
||||
nodes = self._parser.get_clickable_nodes(root)
|
||||
return [self._translate_node(n) for n in nodes]
|
||||
|
||||
def _grid_fast_path(self, intent_description: str, nodes: list, skip_positions: set = None) -> Optional[dict]:
|
||||
if skip_positions is None:
|
||||
skip_positions = set()
|
||||
|
||||
if "first image in explore grid" in intent_description.lower():
|
||||
grid_items = [
|
||||
n
|
||||
for n in nodes
|
||||
if n.get("y", 9999) < 2000
|
||||
and (
|
||||
"grid card layout container" in (n.get("semantic_string", "") or "").lower()
|
||||
or "image button" in (n.get("semantic_string", "") or "").lower()
|
||||
)
|
||||
and (n.get("x", -1), n.get("y", -1)) not in skip_positions
|
||||
]
|
||||
if grid_items:
|
||||
# Sort by y (row) then by x (col)
|
||||
grid_items.sort(key=lambda n: (n.get("y", 9999), n.get("x", 9999)))
|
||||
return grid_items[0]
|
||||
return None
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Action Memory Delegation
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -178,11 +168,11 @@ class TelepathicEngine:
|
||||
def decay_click(self, intent: str = None):
|
||||
self._memory.reject_click(intent) # Alias to reject
|
||||
|
||||
def verify_success(self, intent: str, post_click_xml: str) -> bool:
|
||||
def verify_success(self, intent: str, post_click_xml: str, device=None, confidence: float = 0.0) -> bool:
|
||||
pre_click_xml = ""
|
||||
if self._memory._last_click_context:
|
||||
pre_click_xml = self._memory._last_click_context.get("xml_context", "")
|
||||
return self._memory.verify_success(intent, pre_click_xml, post_click_xml)
|
||||
return self._memory.verify_success(intent, pre_click_xml, post_click_xml, device=device, confidence=confidence)
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Semantic Evaluator Delegation
|
||||
@@ -238,35 +228,34 @@ class TelepathicEngine:
|
||||
y = node.get("y", 0)
|
||||
semantic = (node.get("semantic_string", "") or "").lower()
|
||||
|
||||
# 1. Navigation Tab Guard (Must be at the bottom)
|
||||
nav_intents = [
|
||||
"tap direct message icon inbox",
|
||||
"tap inbox",
|
||||
"tap heart icon notifications",
|
||||
"tap home tab",
|
||||
"tap explore tab",
|
||||
"tap reels tab",
|
||||
"tap profile tab",
|
||||
"tap messages tab",
|
||||
]
|
||||
is_nav_intent = any(n in intent for n in nav_intents)
|
||||
if is_nav_intent:
|
||||
if y < screen_height * 0.85:
|
||||
return False
|
||||
return True
|
||||
|
||||
# 2. Block non-nav intents from clicking in the nav zone
|
||||
if y >= screen_height * 0.85:
|
||||
# Not a nav intent, but trying to click the nav bar
|
||||
return False
|
||||
|
||||
# 3. Post Username Guard
|
||||
if "post username" in intent:
|
||||
if "story" in semantic and y < screen_height * 0.2:
|
||||
# 1. Post Username Guard
|
||||
if "post username" in intent or "author username" in intent:
|
||||
if "story" in semantic:
|
||||
# E.g. "Your Story" circle at the top
|
||||
return False
|
||||
# Prevent tapping a search list item when looking for a post username
|
||||
if "row search user container" in semantic.replace("_", " "):
|
||||
return False
|
||||
# Prevent tapping bottom tabs
|
||||
if "tab" in semantic and "exclude bottom tabs" in intent:
|
||||
return False
|
||||
return True
|
||||
|
||||
# 3.5 Media Content Guard
|
||||
if "post media content" in intent:
|
||||
# Prevent tapping a search keyword instead of a media post
|
||||
if "row search keyword title" in semantic.replace("_", " "):
|
||||
return False
|
||||
# Prevent tapping bottom tabs
|
||||
if "tab" in semantic and "exclude bottom tabs" in intent:
|
||||
return False
|
||||
|
||||
# 3.6 Post Author Username Header Guard
|
||||
if "post author username header" in intent:
|
||||
# Prevent tapping the follow button when looking for the username
|
||||
if "follow button" in semantic.replace("_", " "):
|
||||
return False
|
||||
|
||||
# 4. Profile Picture/Story Ring Guard
|
||||
if "story ring" in intent or "avatar" in intent:
|
||||
current_user = self._get_current_username()
|
||||
|
||||
@@ -65,8 +65,29 @@ def _run_zero_latency_unfollow_loop(
|
||||
try:
|
||||
xml_dump = device.dump_hierarchy()
|
||||
|
||||
# Smart Unfollow Phase 1: Find user rows instead of just clicking "Following"
|
||||
nodes = telepathic._extract_semantic_nodes(xml_dump, "find user profile rows in list", threshold=0.7)
|
||||
# ── Perimeter Guard: Verify we're still inside Instagram ──
|
||||
if xml_dump:
|
||||
import re
|
||||
|
||||
unfollow_packages = set(re.findall(r'package="([^"]+)"', xml_dump))
|
||||
unfollow_app_id = getattr(device, "app_id", "com.instagram.android")
|
||||
if unfollow_packages and unfollow_app_id not in unfollow_packages:
|
||||
logger.error(
|
||||
f"🚨 [UnfollowLoop] FOREIGN APP DETECTED! Packages: {unfollow_packages}. Aborting loop."
|
||||
)
|
||||
device.press("back")
|
||||
random_sleep(1.0, 1.5)
|
||||
return "CONTEXT_LOST"
|
||||
|
||||
# Autonomously identify user rows via Semantic Extraction
|
||||
telepathic = cognitive_stack.get("telepathic")
|
||||
nodes = []
|
||||
if telepathic:
|
||||
nodes = telepathic._extract_semantic_nodes(
|
||||
xml_dump, "List item containing a user profile image, username, and following/following button"
|
||||
)
|
||||
else:
|
||||
logger.warning("No telepathic engine found, skipping semantic extraction.")
|
||||
|
||||
action_taken = False
|
||||
for node in nodes:
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from time import sleep
|
||||
|
||||
@@ -95,6 +97,62 @@ def get_value(count, name, default=0):
|
||||
return default
|
||||
|
||||
|
||||
_LEARNED_AD_MARKERS_FILE = os.path.join(os.getcwd(), "learned_ad_markers.json")
|
||||
_LEARNED_AD_MARKERS_CACHE = None
|
||||
|
||||
def get_learned_ad_markers() -> set:
|
||||
global _LEARNED_AD_MARKERS_CACHE
|
||||
if _LEARNED_AD_MARKERS_CACHE is not None:
|
||||
return _LEARNED_AD_MARKERS_CACHE
|
||||
|
||||
if os.path.exists(_LEARNED_AD_MARKERS_FILE):
|
||||
try:
|
||||
with open(_LEARNED_AD_MARKERS_FILE, "r") as f:
|
||||
_LEARNED_AD_MARKERS_CACHE = set(json.load(f))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load learned ad markers: {e}")
|
||||
_LEARNED_AD_MARKERS_CACHE = set()
|
||||
else:
|
||||
_LEARNED_AD_MARKERS_CACHE = set()
|
||||
|
||||
return _LEARNED_AD_MARKERS_CACHE
|
||||
|
||||
def learn_ad_marker(marker: str, xml_hierarchy: str):
|
||||
global _LEARNED_AD_MARKERS_CACHE
|
||||
if not marker or len(marker) > 30:
|
||||
return
|
||||
|
||||
marker = marker.strip().lower()
|
||||
|
||||
# Structural verification: the VLM-suggested marker MUST exist as an exact node text/desc in the current UI!
|
||||
import xml.etree.ElementTree as ET
|
||||
try:
|
||||
root = ET.fromstring(xml_hierarchy)
|
||||
found_in_ui = False
|
||||
for node in root.iter("node"):
|
||||
text = node.attrib.get("text", "").strip().lower()
|
||||
desc = node.attrib.get("content-desc", "").strip().lower()
|
||||
if text == marker or desc == marker:
|
||||
found_in_ui = True
|
||||
break
|
||||
|
||||
if not found_in_ui:
|
||||
logger.debug(f"🧠 [Autonomous FSD] Rejected hallucinated Ad marker '{marker}' (not found as exact node match in UI).")
|
||||
return
|
||||
except Exception:
|
||||
return
|
||||
|
||||
markers = get_learned_ad_markers()
|
||||
if marker not in markers and marker not in {"ad", "sponsored", "advertisement", "gesponsert", "anzeige", "werbung"}:
|
||||
markers.add(marker)
|
||||
logger.info(f"🧠 [Autonomous FSD] Verified and Learned new Ad marker: '{marker}'. Persisting for zero-latency detection.", extra={"color": f"{Style.BRIGHT}{Fore.GREEN}"})
|
||||
try:
|
||||
with open(_LEARNED_AD_MARKERS_FILE, "w") as f:
|
||||
json.dump(list(markers), f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save learned ad markers: {e}")
|
||||
|
||||
|
||||
def is_ad(xml_hierarchy: str, cognitive_stack: dict = None) -> bool:
|
||||
"""
|
||||
Checks if the current view contains an advertisement using autonomous learning.
|
||||
@@ -102,7 +160,6 @@ def is_ad(xml_hierarchy: str, cognitive_stack: dict = None) -> bool:
|
||||
If a cognitive_stack is provided, it uses the Telepathic Engine for
|
||||
semantic classification (Zero-Latency vector lookup).
|
||||
"""
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
if cognitive_stack:
|
||||
@@ -123,24 +180,36 @@ def is_ad(xml_hierarchy: str, cognitive_stack: dict = None) -> bool:
|
||||
"com.instagram.android:id/ad_not_interested_button",
|
||||
]
|
||||
|
||||
AD_MARKERS = [r"\b(sponsored|ad|advertisement)\b", r"\b(gesponsert|anzeige|werbung)\b"]
|
||||
# Standalone label patterns: match only when the text/desc IS the ad marker,
|
||||
# not when "ad" appears inside longer phrases like "Create messaging ad"
|
||||
AD_EXACT_LABELS = {"ad", "sponsored", "advertisement", "gesponsert", "anzeige", "werbung"}
|
||||
AD_EXACT_LABELS.update(get_learned_ad_markers())
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_hierarchy)
|
||||
|
||||
# Check if we are in a feed (to prevent false positives on profiles with 'Ad Tools' buttons)
|
||||
from GramAddict.core.perception.feed_analysis import FEED_MARKERS
|
||||
in_feed = any(marker in xml_hierarchy for marker in FEED_MARKERS)
|
||||
|
||||
for node in root.iter("node"):
|
||||
attrib = node.attrib
|
||||
content_desc = attrib.get("content-desc", "")
|
||||
text = attrib.get("text", "")
|
||||
res_id = attrib.get("resource-id", "")
|
||||
|
||||
# Structural check (Instagram specific)
|
||||
# Structural check (Instagram specific) is always trusted
|
||||
if any(marker_id in res_id for marker_id in AD_RESOURCE_IDS):
|
||||
return True
|
||||
|
||||
# Content check (Legacy)
|
||||
searchable = f"{content_desc} {text}".lower()
|
||||
for pattern in AD_MARKERS:
|
||||
if re.search(pattern, searchable):
|
||||
# Exact label match: only trigger when the entire text/desc
|
||||
# IS an ad marker (e.g. text="Ad", content-desc="Sponsored")
|
||||
# We ONLY trust this if we are actually in a feed, to prevent triggering
|
||||
# on the "Ad Tools" / "Ad" buttons present on business profiles.
|
||||
if in_feed:
|
||||
if text.strip().lower() in AD_EXACT_LABELS:
|
||||
return True
|
||||
if content_desc.strip().lower() in AD_EXACT_LABELS:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -21,6 +21,7 @@ If Instagram updates its app and moves a button, GramPilot doesn't crash. It fal
|
||||
## ✨ Core Features
|
||||
|
||||
* 🚫 **Zero Limits Configuration**: Forget about configuring "max_likes" or "delays". GramPilot uses a **Dopamine Pacing Engine** to simulate human boredom. If the content isn't interesting, it skips it or ends the session early.
|
||||
* 🎯 **Mission-Driven Navigation**: Say goodbye to abstract goal configurations. Define a `strategy` (like `aggressive_growth` or `nurture_community`) in `config.yml`, and the **Goal Decomposer Engine** automatically orchestrates the optimal routing and task allocation using enabled plugins.
|
||||
* ⚖️ **Active Inference (Shadow Mode)**: The bot continuously predicts the outcome of its clicks. If it lands on a popup instead of a profile, it registers a "Prediction Error", presses back, and dynamically recalibrates without panicking.
|
||||
* ⛩️ **Telepathic Engine**: A strictly tiered resolution cascade (Keyword -> Vectors -> LLM) that ensures 90% of navigation happens at 0-token cost while maintaining fallback AI resilience.
|
||||
* 🧬 **Resonance Oracle**: The bot only interacts with content that matches a pre-defined persona aesthetic, completely bypassing spam or low-quality content.
|
||||
|
||||
@@ -9,11 +9,14 @@ from datetime import datetime
|
||||
# Add root project path so we can import internal modules safely
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
from GramAddict.core.llm_provider import query_llm, query_telepathic_llm
|
||||
|
||||
BENCHMARKS_FILE = os.path.join(os.path.dirname(__file__), "data/llm_benchmarks.json")
|
||||
SCENARIOS_FILE = os.path.join(os.path.dirname(__file__), "data/benchmark_scenarios.json")
|
||||
|
||||
# Minimum iterations for statistical significance
|
||||
MIN_ITERATIONS = 5
|
||||
|
||||
|
||||
def load_json(path):
|
||||
if os.path.exists(path):
|
||||
@@ -31,35 +34,37 @@ def save_json(path, data):
|
||||
|
||||
|
||||
def normalize_scores(db):
|
||||
"""Normalize relative performance by AVERAGE score per scenario, not raw totals."""
|
||||
if not db.get("models"):
|
||||
return db
|
||||
|
||||
# 1. Find the highest raw score across all models
|
||||
max_raw = 0
|
||||
max_avg = 0
|
||||
leader_model = None
|
||||
|
||||
for name, data in db["models"].items():
|
||||
if data.get("is_unsuitable"):
|
||||
continue
|
||||
|
||||
raw = data.get("raw_score", 0)
|
||||
if raw > max_raw:
|
||||
max_raw = raw
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["avg_score_per_scenario"] = round(avg, 1)
|
||||
|
||||
if avg > max_avg:
|
||||
max_avg = avg
|
||||
leader_model = name
|
||||
elif raw == max_raw and max_raw > 0:
|
||||
# Tie-breaker: Latency
|
||||
elif avg == max_avg and max_avg > 0:
|
||||
current_lat = data.get("latency_ms", 99999)
|
||||
leader_lat = db["models"][leader_model].get("latency_ms", 99999)
|
||||
if current_lat < leader_lat:
|
||||
leader_model = name
|
||||
|
||||
if max_raw == 0:
|
||||
if max_avg == 0:
|
||||
return db
|
||||
|
||||
# 2. Update relative performance
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
|
||||
scenario_count = data.get("scenario_count", 1)
|
||||
avg = data.get("raw_score", 0) / max(scenario_count, 1)
|
||||
data["relative_performance_pct"] = round((avg / max_avg) * 100, 1)
|
||||
data["is_leader"] = name == leader_model
|
||||
|
||||
return db
|
||||
@@ -75,21 +80,15 @@ def get_installed_ollama_models():
|
||||
models = []
|
||||
for line in output.split("\n")[1:]:
|
||||
if line.strip():
|
||||
# Format: NAME, ID, SIZE, MODIFIED
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
name = parts[0]
|
||||
size = parts[2]
|
||||
|
||||
# 1. Skip if size is '-' (remote/cloud model)
|
||||
if size == "-":
|
||||
continue
|
||||
|
||||
# 2. Skip ':cloud' tagged models explicitly
|
||||
if ":cloud" in name:
|
||||
continue
|
||||
|
||||
# 3. Filter out purely embedding models
|
||||
if any(k in name.lower() for k in ["embed", "minilm", "rerank"]):
|
||||
continue
|
||||
|
||||
@@ -100,7 +99,131 @@ def get_installed_ollama_models():
|
||||
return []
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False):
|
||||
def _run_telepathic_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a telepathic (JSON element selection) scenario."""
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
'Return: {"index": number, "reason": "..."}'
|
||||
)
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def _run_brain_scenario(scenario, model_name, url, iterations):
|
||||
"""Run a brain action extraction scenario (format_json=False)."""
|
||||
system_prompt = (
|
||||
f"You are an autonomous Instagram agent. Your goal is: '{scenario['task']}'.\n"
|
||||
f"You are currently on screen: {scenario['screen_type']}.\n"
|
||||
f"Available actions: {scenario['available_actions']}\n"
|
||||
"INSTRUCTIONS: Reply with ONLY the action string. Nothing else."
|
||||
)
|
||||
|
||||
user_prompt = "Choose the next best action."
|
||||
|
||||
latencies = []
|
||||
scores = []
|
||||
successes = 0
|
||||
|
||||
for _ in range(iterations):
|
||||
start_time = time.time()
|
||||
try:
|
||||
# CRITICAL: Use format_json=False — this is the Brain code path
|
||||
ans = query_llm(
|
||||
url=url,
|
||||
model=model_name,
|
||||
prompt=user_prompt,
|
||||
system=system_prompt,
|
||||
format_json=False,
|
||||
timeout=30,
|
||||
temperature=0.0,
|
||||
max_tokens=50,
|
||||
)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
latencies.append(latency)
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed: {e}")
|
||||
scores.append(0)
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
if ans and "response" in ans:
|
||||
response = ans["response"].strip().lower()
|
||||
|
||||
# Points for structural adherence (returned a clean string)
|
||||
if response and response in [a.lower() for a in scenario["available_actions"]]:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if scenario.get("accept_any_valid"):
|
||||
# Any valid action from the list is acceptable
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
elif response == scenario["target_action"].lower():
|
||||
raw_points += 60
|
||||
successes += 1
|
||||
else:
|
||||
print(f" ⚠️ Valid but suboptimal: '{response}' (target: '{scenario['target_action']}')")
|
||||
raw_points += 20 # Partial credit for valid but wrong action
|
||||
else:
|
||||
print(f" ❌ Invalid response: '{response}' not in available actions")
|
||||
else:
|
||||
print(" ❌ Empty or null response from LLM")
|
||||
|
||||
scores.append(raw_points)
|
||||
|
||||
return scores, latencies, successes
|
||||
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = MIN_ITERATIONS):
|
||||
iterations = max(iterations, MIN_ITERATIONS) # Enforce minimum
|
||||
|
||||
db = load_json(BENCHMARKS_FILE) or {"models": {}}
|
||||
scenarios_data = load_json(SCENARIOS_FILE)
|
||||
if not scenarios_data:
|
||||
@@ -113,75 +236,46 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
|
||||
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
|
||||
return
|
||||
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name}")
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name} ({iterations} iterations)")
|
||||
|
||||
total_raw = 0
|
||||
total_latency = 0
|
||||
results_detail = {}
|
||||
passed_all = True
|
||||
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
|
||||
)
|
||||
|
||||
scenarios = scenarios_data["scenarios"]
|
||||
for scenario in scenarios:
|
||||
print(f"--- Running: {scenario['name']} ---")
|
||||
scenario_type = scenario.get("type", "telepathic")
|
||||
print(f"--- [{scenario_type.upper()}] {scenario['name']} ---")
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
"Return: {\"index\": number, \"reason\": \"...\"}"
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
total_latency += latency
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed for scenario {scenario['id']}: {e}")
|
||||
passed_all = False
|
||||
if scenario_type == "telepathic":
|
||||
scores, latencies, successes = _run_telepathic_scenario(scenario, model_name, url, iterations)
|
||||
elif scenario_type == "brain_action":
|
||||
scores, latencies, successes = _run_brain_scenario(scenario, model_name, url, iterations)
|
||||
else:
|
||||
print(f" ⚠️ Unknown scenario type: {scenario_type}")
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"):
|
||||
clean = clean[7:]
|
||||
if clean.endswith("```"):
|
||||
clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
avg_score = int(sum(scores) / len(scores)) if scores else 0
|
||||
avg_latency = int(sum(latencies) / len(latencies)) if latencies else 0
|
||||
pass_rate = (successes / iterations) * 100
|
||||
|
||||
# Points for structural adherence
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
print(f" ✅ Correct index ({data['index']}).")
|
||||
else:
|
||||
passed_all = False
|
||||
print(f" ❌ Wrong index ({data['index']}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
passed_all = False
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
if pass_rate < 100.0:
|
||||
passed_all = False
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
results_detail[scenario["id"]] = raw_points
|
||||
total_raw += raw_points
|
||||
print(f" Result: {pass_rate:.0f}% Pass | Avg Score: {avg_score}/100 | Avg Latency: {avg_latency}ms")
|
||||
|
||||
# Consistent format: always an object
|
||||
results_detail[scenario["id"]] = {
|
||||
"avg_score": avg_score,
|
||||
"pass_rate": pass_rate,
|
||||
"latency": avg_latency,
|
||||
}
|
||||
total_raw += avg_score
|
||||
total_latency += avg_latency
|
||||
|
||||
avg_latency = total_latency // len(scenarios) if scenarios else 0
|
||||
print(
|
||||
f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Score: {total_raw} | Latency: {avg_latency}ms"
|
||||
)
|
||||
print(f"\n📊 {model_name}: {'PASS' if passed_all else 'FAIL'} | Total: {total_raw} | Latency: {avg_latency}ms")
|
||||
|
||||
if model_name not in db["models"]:
|
||||
db["models"][model_name] = {}
|
||||
@@ -189,16 +283,17 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
|
||||
db["models"][model_name].update(
|
||||
{
|
||||
"raw_score": total_raw,
|
||||
"scenario_count": len(scenarios),
|
||||
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
|
||||
"latency_ms": avg_latency,
|
||||
"last_tested": datetime.utcnow().isoformat() + "Z",
|
||||
"details": results_detail,
|
||||
"passed_all": passed_all,
|
||||
"is_unsuitable": not passed_all,
|
||||
"iterations": iterations,
|
||||
}
|
||||
)
|
||||
|
||||
# Recalculate relative scores across all models
|
||||
db = normalize_scores(db)
|
||||
save_json(BENCHMARKS_FILE, db)
|
||||
|
||||
@@ -212,6 +307,9 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--url", type=str, help="Explicit endpoint URL")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-testing")
|
||||
parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=MIN_ITERATIONS, help=f"Iterations per scenario (min: {MIN_ITERATIONS})"
|
||||
)
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
@@ -241,5 +339,5 @@ if __name__ == "__main__":
|
||||
sys.exit(1)
|
||||
|
||||
for m, u in set(models_to_test):
|
||||
benchmark_model(m, u, args.force)
|
||||
benchmark_model(m, u, args.force, args.iterations)
|
||||
time.sleep(1)
|
||||
|
||||
@@ -89,6 +89,11 @@ telegram-reports: false # for using telegram-reports you have also to configure
|
||||
interactions-count: 30-40
|
||||
likes-count: 1-2
|
||||
likes-percentage: 100
|
||||
|
||||
plugins:
|
||||
dm_reply:
|
||||
enabled: false # Generates AI replies to unread DMs
|
||||
|
||||
stories-count: 1-2
|
||||
stories-percentage: 30-40
|
||||
carousel-count: 2-3
|
||||
|
||||
@@ -7,7 +7,7 @@ emoji==2.12.1
|
||||
langdetect==1.0.9
|
||||
atomicwrites==1.4.1
|
||||
spintax==1.0.4
|
||||
requests>=2.31.0
|
||||
requests>=2.32.0
|
||||
packaging>=23.0
|
||||
python-dotenv==1.0.1
|
||||
qdrant-client>=1.7.0
|
||||
|
||||
@@ -20,13 +20,32 @@ else
|
||||
filename=$(basename "$file")
|
||||
# Heuristic: Try to find a matching unit test
|
||||
test_file="tests/unit/test_${filename}"
|
||||
core_test_file="tests/core/test_${filename}"
|
||||
if [ -f "$test_file" ]; then
|
||||
TEST_TARGETS="$TEST_TARGETS $test_file"
|
||||
elif [ -f "$core_test_file" ]; then
|
||||
TEST_TARGETS="$TEST_TARGETS $core_test_file"
|
||||
else
|
||||
# If no direct unit test, fallback to running all unit tests to be safe
|
||||
echo "⚠️ No direct unit test found for $file, falling back to all unit tests."
|
||||
TEST_TARGETS="tests/unit"
|
||||
break
|
||||
# Try to find matching e2e tests by searching for each word in the module name
|
||||
module_name="${filename%.py}"
|
||||
e2e_matches=""
|
||||
for word in $(echo "$module_name" | tr '_' '\n'); do
|
||||
if [ ${#word} -ge 4 ]; then # Only search meaningful words (4+ chars)
|
||||
found=$(find tests/e2e -name "test_*${word}*.py" 2>/dev/null | head -3)
|
||||
if [ -n "$found" ]; then
|
||||
e2e_matches="$e2e_matches $found"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
e2e_matches=$(echo "$e2e_matches" | xargs -n1 2>/dev/null | sort -u | head -3 | xargs 2>/dev/null)
|
||||
if [ -n "$e2e_matches" ]; then
|
||||
echo "⚠️ No direct unit test for $file, using matching E2E tests: $e2e_matches"
|
||||
TEST_TARGETS="$TEST_TARGETS $e2e_matches"
|
||||
else
|
||||
echo "⚠️ No direct unit test found for $file, falling back to all unit tests."
|
||||
TEST_TARGETS="tests/unit"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@@ -41,7 +60,7 @@ if [ -z "$TEST_TARGETS" ]; then
|
||||
fi
|
||||
|
||||
echo "🧪 Running tests on: $TEST_TARGETS"
|
||||
venv/bin/pytest $TEST_TARGETS --cov=GramAddict --cov-report=xml -q
|
||||
PYTHONPATH=. venv/bin/pytest $TEST_TARGETS --cov=GramAddict --cov-report=xml -q
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
|
||||
@@ -18,8 +18,8 @@ logger = logging.getLogger("TestingToolkit")
|
||||
def _save_dump(device, fixture_dir, filename, description):
|
||||
logger.info(f"⏳ Waiting for UI to settle for [{description}]...")
|
||||
time.sleep(3.5) # ensure animations finish
|
||||
xml_data = device.dump_hierarchy()
|
||||
|
||||
xml_data = device.dump_hierarchy()
|
||||
if not xml_data or len(xml_data) < 100:
|
||||
logger.warning(f"⚠️ Received empty or exceptionally small XML dump for {filename}. Is the app open?")
|
||||
|
||||
@@ -28,6 +28,23 @@ def _save_dump(device, fixture_dir, filename, description):
|
||||
f.write(xml_data)
|
||||
logger.info(f"✅ Saved REAL DUMP to {filename} ({len(xml_data)} bytes)")
|
||||
|
||||
# Capture screenshot
|
||||
try:
|
||||
import base64
|
||||
|
||||
screenshot_b64 = device.get_screenshot_b64()
|
||||
if screenshot_b64:
|
||||
screenshot_data = base64.b64decode(screenshot_b64)
|
||||
screenshot_filename = filename.replace(".xml", ".jpg")
|
||||
screenshot_path = os.path.join(fixture_dir, screenshot_filename)
|
||||
with open(screenshot_path, "wb") as f:
|
||||
f.write(screenshot_data)
|
||||
logger.info(f"✅ Saved REAL SCREENSHOT to {screenshot_filename}")
|
||||
else:
|
||||
logger.warning(f"⚠️ Failed to capture screenshot for {filename}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to capture screenshot: {e}")
|
||||
|
||||
|
||||
def run_interactive_guide(device, fixture_dir):
|
||||
print("\n" + "=" * 60)
|
||||
@@ -79,6 +96,13 @@ def main():
|
||||
|
||||
device_id = args.device
|
||||
|
||||
# Auto-detect config if not provided
|
||||
if not args.config:
|
||||
if os.path.exists("test_config.yml"):
|
||||
args.config = "test_config.yml"
|
||||
elif os.path.exists("config.yml"):
|
||||
args.config = "config.yml"
|
||||
|
||||
# Try to extract device from config if provided
|
||||
if args.config:
|
||||
try:
|
||||
|
||||
107
test_config.yml
107
test_config.yml
@@ -1,107 +0,0 @@
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 🤖 ANTIGRAVITY ELITE CONFIGURATION (Plugin-Based Architecture)
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# Dieses Brain ist modular aufgebaut. Jedes Verhalten ist ein autonomes Plugin.
|
||||
# Einstellungen können global oder spezifisch für jedes Plugin definiert werden.
|
||||
# Design-Prinzip: Zero Trust & Fail Fast.
|
||||
|
||||
identity:
|
||||
username: "marisaundmarc"
|
||||
persona: "Travel blogger, landscape photographer, and outdoors enthusiast"
|
||||
vibe: "friendly, authentic, helpful, and appreciative of good art"
|
||||
|
||||
mission:
|
||||
strategy: "aggressive_growth"
|
||||
selectivity_threshold: "high"
|
||||
target_audience: "travel, landscape, nature, mountain photography, wanderlust"
|
||||
blacklist_topics: "onlyfans, nsfw, sale, discount, promo, 18+, giveaway, crypto"
|
||||
|
||||
# ── Core Action Jobs (Wann soll der Bot wo aktiv werden?) ──
|
||||
actions:
|
||||
feed: "5-10" # Anzahl der Posts im Home-Feed pro Session
|
||||
explore: "5-10" # Anzahl der Posts im Explore-Grid
|
||||
# reels: "5-10" # In Entwicklung
|
||||
# stories: "3-5" # In Entwicklung
|
||||
|
||||
# ── Plugin Configuration (Das Herzstück der Verhaltenssteuerung) ──
|
||||
plugins:
|
||||
# 🛡️ Guards & Safety (Filtern, bevor Interaktion passiert)
|
||||
ad_guard:
|
||||
enabled: true
|
||||
|
||||
close_friends_guard:
|
||||
enabled: true # Postings von 'Engen Freunden' ignorieren
|
||||
|
||||
obstacle_guard:
|
||||
enabled: true # Popups, Update-Dialoge etc. wegräumen
|
||||
|
||||
anomaly_handler:
|
||||
enabled: true # Erkennt Blockierungen oder Captchas sofort (Fail Fast)
|
||||
|
||||
# 🧠 Perception & Evaluation (Vorverarbeitung)
|
||||
post_data_extraction:
|
||||
enabled: true # Extrahiert Text, Hashtags und Metadata
|
||||
|
||||
resonance_evaluator:
|
||||
visual_vibe_check_percentage: 100
|
||||
selectivity_threshold: "high"
|
||||
|
||||
# ⚡ Interactions (Die eigentlichen Aktionen)
|
||||
likes:
|
||||
percentage: 100 # Wahrscheinlichkeit pro Post
|
||||
count: "2-3" # Falls im Grid, wie viele?
|
||||
|
||||
comment:
|
||||
percentage: 40
|
||||
dry_run: true # Generiert KI-Kommentare ohne zu posten (Review-Mode)
|
||||
|
||||
follow:
|
||||
percentage: 100
|
||||
|
||||
repost:
|
||||
percentage: 20 # Teilen in die eigene Story
|
||||
|
||||
story_view:
|
||||
percentage: 80
|
||||
count: "1-3" # Wie viele Stories pro User schauen?
|
||||
|
||||
profile_visit:
|
||||
percentage: 100 # Wahrscheinlichkeit, vom Feed ins Profil zu gehen
|
||||
learn_own_profile: true
|
||||
|
||||
grid_like:
|
||||
percentage: 60 # Liked Posts aus dem Profil-Grid des Users
|
||||
count: "1-3"
|
||||
|
||||
# 🎢 Special Behaviors
|
||||
carousel_browsing:
|
||||
percentage: 100 # Erkennt Carousels und swiped durch
|
||||
count: "2-4" # Wie viele Slides pro Post?
|
||||
|
||||
rabbit_hole:
|
||||
percentage: 30 # Geht tiefer in verwandte Profile (Inception-Mode)
|
||||
|
||||
darwin_dwell:
|
||||
percentage: 50 # Simuliert unregelmäßige Lesezeiten (Biometrie)
|
||||
|
||||
# ── Limits & Budget ──
|
||||
limits:
|
||||
daily_budget_hours: 2.5
|
||||
max_comments_per_day: 40
|
||||
total_likes_limit: 300
|
||||
total_follows_limit: 50
|
||||
speed_multiplier: 1.0
|
||||
|
||||
# ── Infrastructure & System ──
|
||||
device: 192.168.1.206:40505
|
||||
app-id: com.instagram.android
|
||||
debug: true
|
||||
blank_start: true
|
||||
|
||||
# ── AI Model Endpoints (Ollama / OpenRouter) ──
|
||||
ai-model: qwen3.5:latest
|
||||
ai-model-url: http://localhost:11434/api/generate
|
||||
ai-telepathic-model: llama3.2-vision
|
||||
ai-telepathic-url: http://localhost:11434/api/generate
|
||||
ai-embedding-model: nomic-embed-text
|
||||
ai-embedding-url: http://localhost:11434/api/embeddings
|
||||
621
test_errors.txt
Normal file
621
test_errors.txt
Normal file
@@ -0,0 +1,621 @@
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.11.9, pytest-8.3.5, pluggy-1.5.0 -- /Users/marcmintel/.pyenv/versions/3.11.9/bin/python3
|
||||
cachedir: .pytest_cache
|
||||
hypothesis profile 'default'
|
||||
metadata: {'Python': '3.11.9', 'Platform': 'macOS-26.3.1-arm64-arm-64bit', 'Packages': {'pytest': '8.3.5', 'pluggy': '1.5.0'}, 'Plugins': {'anyio': '4.8.0', 'snapshot': '0.9.0', 'xdist': '3.7.0', 'instafail': '0.5.0', 'allure-pytest': '2.15.0', 'hypothesis': '6.140.2', 'html': '4.1.1', 'json-report': '1.5.0', 'timeout': '2.4.0', 'metadata': '3.1.1', 'md': '0.2.0', 'Faker': '37.8.0', 'clarity': '1.0.1', 'datadir': '1.8.0', 'cov': '6.2.1', 'mock': '3.14.1', 'pytest_httpserver': '1.1.3', 'sugar': '1.1.1', 'benchmark': '5.1.0', 'rerunfailures': '16.0.1'}}
|
||||
benchmark: 5.1.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
|
||||
rootdir: /Volumes/Alpha SSD/Coding/bot
|
||||
configfile: pyproject.toml
|
||||
plugins: anyio-4.8.0, snapshot-0.9.0, xdist-3.7.0, instafail-0.5.0, allure-pytest-2.15.0, hypothesis-6.140.2, html-4.1.1, json-report-1.5.0, timeout-2.4.0, metadata-3.1.1, md-0.2.0, Faker-37.8.0, clarity-1.0.1, datadir-1.8.0, cov-6.2.1, mock-3.14.1, pytest_httpserver-1.1.3, sugar-1.1.1, benchmark-5.1.0, rerunfailures-16.0.1
|
||||
collecting ... collected 186 items
|
||||
|
||||
tests/anomalies/test_cognitive_edge_cases.py::TestCognitiveEdgeCases::test_resonance_edge_cases PASSED [ 0%]
|
||||
tests/anomalies/test_cognitive_edge_cases.py::TestCognitiveEdgeCases::test_darwin_edge_cases PASSED [ 1%]
|
||||
tests/anomalies/test_cognitive_edge_cases.py::TestCognitiveEdgeCases::test_growth_brain_edge_cases PASSED [ 1%]
|
||||
tests/anomalies/test_hardware_anomalies_gauss.py::test_gaussian_distribution PASSED [ 2%]
|
||||
tests/anomalies/test_telepathic_guards.py::TestTelepathicGuards::test_strict_story_ring_guard FAILED [ 2%]
|
||||
tests/anomalies/test_telepathic_guards.py::TestTelepathicGuards::test_strict_button_guard FAILED [ 3%]
|
||||
tests/anomalies/test_telepathic_guards.py::TestTelepathicGuards::test_like_semantic_verification PASSED [ 3%]
|
||||
tests/anomalies/test_trap_radome.py::test_zero_point_trap PASSED [ 4%]
|
||||
tests/anomalies/test_trap_radome.py::test_micro_pixel_trap PASSED [ 4%]
|
||||
tests/anomalies/test_trap_radome.py::test_safe_normal_button PASSED [ 5%]
|
||||
tests/anomalies/test_trap_radome.py::test_transparent_interceptor_trap PASSED [ 5%]
|
||||
tests/anomalies/test_trap_radome.py::test_accessibility_trap PASSED [ 6%]
|
||||
tests/e2e/test_e2e_animation_timing.py::test_animation_timing_mocks_purged SKIPPED [ 6%]
|
||||
tests/e2e/test_e2e_dm_engine.py::test_e2e_dm_full_flow_success_real SKIPPED [ 7%]
|
||||
tests/e2e/test_e2e_dm_engine.py::test_e2e_dm_no_messages_real SKIPPED [ 8%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_normal_instagram ERROR [ 8%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_foreign_app_google ERROR [ 9%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_notification_shade ERROR [ 9%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_system_permission_dialog ERROR [ 10%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_instagram_survey_modal ERROR [ 10%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_unknown_modal_interstitial ERROR [ 11%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_action_blocked ERROR [ 11%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_empty_dump ERROR [ 12%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_none_dump ERROR [ 12%]
|
||||
tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_passive_scaffold_as_normal ERROR [ 13%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_home_feed_as_normal ERROR [ 13%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_explore_grid_as_normal ERROR [ 14%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_other_profile_as_normal ERROR [ 15%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_post_detail_as_normal ERROR [ 15%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_profile_tagged_tab_as_normal ERROR [ 16%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_survey_modal_as_obstacle ERROR [ 16%]
|
||||
tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_mystery_interstitial_as_obstacle ERROR [ 17%]
|
||||
tests/e2e/test_engine_perception.py::test_perception_mock_theater_purged SKIPPED [ 17%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_goap_planner_avoids_infinite_loop_on_masked_edge ERROR [ 18%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_screen_topology_find_route_avoids_blocked_edges ERROR [ 18%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_telepathic_engine_finds_following_node_on_profile ERROR [ 19%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_following_vs_followers_are_both_candidates ERROR [ 19%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_vlm_prompt_humanizes_content_desc ERROR [ 20%]
|
||||
tests/e2e/test_goap_loop_prevention.py::test_live_vlm_selects_following_not_followers ERROR [ 20%]
|
||||
tests/e2e/test_reel_interactions.py::test_reel_like_button_not_caption ERROR [ 21%]
|
||||
tests/e2e/test_reel_interactions.py::test_reel_follow_button_returns_none_when_absent ERROR [ 22%]
|
||||
tests/e2e/test_reel_interactions.py::test_reel_post_author_selects_username ERROR [ 22%]
|
||||
tests/e2e/test_reel_interactions.py::test_reel_dedup_preserves_like_button ERROR [ 23%]
|
||||
tests/e2e/test_reel_interactions.py::test_reel_caption_with_like_word_is_not_like_button ERROR [ 23%]
|
||||
tests/e2e/test_reel_navigation_guards.py::test_intent_resolver_profile_tab_rejects_author_profile ERROR [ 24%]
|
||||
tests/e2e/test_reel_navigation_guards.py::test_intent_resolver_profile_tab_selects_real_tab ERROR [ 24%]
|
||||
tests/e2e/test_sim_full_lifecycle.py::test_full_lifecycle_sim_purged SKIPPED [ 25%]
|
||||
tests/e2e/test_visual_intent_resolver.py::test_visual_discovery_creates_annotated_screenshot ERROR [ 25%]
|
||||
tests/e2e/test_visual_intent_resolver.py::test_visual_discovery_finds_following_by_seeing ERROR [ 26%]
|
||||
tests/e2e/test_visual_intent_resolver.py::test_resolve_uses_visual_discovery_when_device_available ERROR [ 26%]
|
||||
tests/integration/test_ad_detection.py::test_real_sponsored_reel_flexcode_is_detected PASSED [ 27%]
|
||||
tests/integration/test_ad_detection.py::test_normal_post_not_ad PASSED [ 27%]
|
||||
tests/integration/test_ad_detection.py::test_peugeot_carousel_ad_is_detected PASSED [ 28%]
|
||||
tests/integration/test_core_nav_dm_regression.py::test_core_nav_rejects_generic_action_bar_right PASSED [ 29%]
|
||||
tests/integration/test_false_positive.py::test_real_normal_post_is_not_ad PASSED [ 29%]
|
||||
tests/integration/test_telepathic_hardening.py::test_keyword_nav_threshold FAILED [ 30%]
|
||||
tests/integration/test_telepathic_hardening.py::test_direct_tab_fast_path FAILED [ 30%]
|
||||
tests/integration/test_telepathic_keyword.py::test_keyword_fast_path_no_feed_pollution FAILED [ 31%]
|
||||
tests/repro_reports/test_repro_api_mismatch.py::TestAPIMismatch::test_repro_extract_semantic_nodes_type_error PASSED [ 31%]
|
||||
tests/repro_reports/test_repro_position_rejection.py::TestPositionRejection::test_repro_following_button_rejection_fix FAILED [ 32%]
|
||||
tests/repro_reports/test_repro_reels_tab_hallucination.py::TestReproReelsTabHallucination::test_reels_tab_selection FAILED [ 32%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_returns_correct_number_of_points PASSED [ 33%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_start_and_end_are_near_requested_positions PASSED [ 33%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_path_is_non_linear PASSED [ 34%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_right_hander_arcs_right PASSED [ 34%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_left_hander_arcs_left PASSED [ 35%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_pressure_has_gaussian_peak PASSED [ 36%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_pressure_within_valid_range PASSED [ 36%]
|
||||
tests/tdd/test_bezier_gesture.py::TestScrollCurve::test_all_points_have_three_components PASSED [ 37%]
|
||||
tests/tdd/test_bezier_gesture.py::TestTapCurve::test_returns_three_points PASSED [ 37%]
|
||||
tests/tdd/test_bezier_gesture.py::TestTapCurve::test_micro_drift_is_small PASSED [ 38%]
|
||||
tests/tdd/test_bezier_gesture.py::TestTapCurve::test_pressure_sequence_is_down_peak_up PASSED [ 38%]
|
||||
tests/tdd/test_bezier_gesture.py::TestHorizontalSwipeCurve::test_returns_reasonable_point_count PASSED [ 39%]
|
||||
tests/tdd/test_bezier_gesture.py::TestHorizontalSwipeCurve::test_horizontal_distance_is_correct_direction PASSED [ 39%]
|
||||
tests/tdd/test_bezier_gesture.py::TestHorizontalSwipeCurve::test_vertical_arc_exists PASSED [ 40%]
|
||||
tests/tdd/test_bezier_gesture.py::TestSigmoidTiming::test_total_duration_matches PASSED [ 40%]
|
||||
tests/tdd/test_bezier_gesture.py::TestSigmoidTiming::test_edges_are_slower_than_middle PASSED [ 41%]
|
||||
tests/tdd/test_bezier_gesture.py::TestSigmoidTiming::test_single_point_returns_single_interval PASSED [ 41%]
|
||||
tests/tdd/test_bezier_gesture.py::TestSigmoidTiming::test_no_negative_intervals PASSED [ 42%]
|
||||
tests/tdd/test_physics_body.py::TestHandedness::test_right_hander_anchor_is_right PASSED [ 43%]
|
||||
tests/tdd/test_physics_body.py::TestHandedness::test_left_hander_anchor_is_left PASSED [ 43%]
|
||||
tests/tdd/test_physics_body.py::TestHandedness::test_right_hander_scroll_starts_right PASSED [ 44%]
|
||||
tests/tdd/test_physics_body.py::TestHandedness::test_left_hander_scroll_starts_left PASSED [ 44%]
|
||||
tests/tdd/test_physics_body.py::TestThumbArcBias::test_right_hander_arcs_right PASSED [ 45%]
|
||||
tests/tdd/test_physics_body.py::TestThumbArcBias::test_left_hander_arcs_left PASSED [ 45%]
|
||||
tests/tdd/test_physics_body.py::TestSessionDrift::test_drift_is_zero_initially PASSED [ 46%]
|
||||
tests/tdd/test_physics_body.py::TestSessionDrift::test_drift_accumulates_over_many_gestures PASSED [ 46%]
|
||||
tests/tdd/test_physics_body.py::TestSessionDrift::test_drift_is_bounded PASSED [ 47%]
|
||||
tests/tdd/test_physics_body.py::TestStartPositions::test_positions_stay_within_screen_bounds PASSED [ 47%]
|
||||
tests/tdd/test_physics_body.py::TestStartPositions::test_positions_are_not_identical PASSED [ 48%]
|
||||
tests/tdd/test_physics_body.py::TestStartPositions::test_gesture_count_increments PASSED [ 48%]
|
||||
tests/tdd/test_physics_body.py::TestFatigue::test_fatigue_starts_at_zero PASSED [ 49%]
|
||||
tests/tdd/test_physics_body.py::TestFatigue::test_rapid_gestures_increase_fatigue PASSED [ 50%]
|
||||
tests/tdd/test_physics_body.py::TestFatigue::test_idle_period_reduces_fatigue PASSED [ 50%]
|
||||
tests/tdd/test_physics_body.py::TestFatigue::test_fatigue_is_clamped_0_to_1 PASSED [ 51%]
|
||||
tests/tdd/test_physics_body.py::TestTapPosition::test_tap_position_near_target PASSED [ 51%]
|
||||
tests/tdd/test_physics_body.py::TestTapPosition::test_tap_stays_on_screen PASSED [ 52%]
|
||||
tests/tdd/test_physics_body.py::TestPressureAndTouchMajor::test_pressure_baseline_in_range PASSED [ 52%]
|
||||
tests/tdd/test_physics_body.py::TestPressureAndTouchMajor::test_fatigue_increases_pressure PASSED [ 53%]
|
||||
tests/tdd/test_physics_body.py::TestPressureAndTouchMajor::test_touch_major_in_range PASSED [ 53%]
|
||||
tests/tdd/test_physics_body.py::TestPressureAndTouchMajor::test_fatigue_increases_touch_major PASSED [ 54%]
|
||||
tests/tdd/test_physics_body.py::TestSingleton::test_singleton_returns_same_instance PASSED [ 54%]
|
||||
tests/tdd/test_physics_body.py::TestSingleton::test_reset_clears_singleton PASSED [ 55%]
|
||||
tests/tdd/test_semantic_heuristic_match.py::test_semantic_heuristic_match_blank_start PASSED [ 55%]
|
||||
tests/unit/perception/test_intent_resolver.py::test_intent_resolver_finds_bottom_tab FAILED [ 56%]
|
||||
tests/unit/perception/test_intent_resolver.py::test_intent_resolver_finds_button_by_text PASSED [ 56%]
|
||||
tests/unit/perception/test_intent_resolver.py::test_intent_resolver_returns_none_if_no_match PASSED [ 57%]
|
||||
tests/unit/perception/test_spatial_parser.py::TestSpatialParser::test_parses_xml_into_spatial_nodes PASSED [ 58%]
|
||||
tests/unit/perception/test_spatial_parser.py::TestSpatialParser::test_extracts_all_clickable_nodes PASSED [ 58%]
|
||||
tests/unit/perception/test_spatial_parser.py::TestSpatialParser::test_spatial_containment PASSED [ 59%]
|
||||
tests/unit/perception/test_spatial_parser.py::TestSpatialParser::test_spatial_intersection PASSED [ 59%]
|
||||
tests/unit/test_config_plugins.py::test_config_plugin_section PASSED [ 60%]
|
||||
tests/unit/test_config_plugins.py::test_config_plugin_fallback PASSED [ 60%]
|
||||
tests/unit/test_config_plugins.py::test_config_plugin_not_found PASSED [ 61%]
|
||||
tests/unit/test_darwin_engine_comments.py::test_has_comments_true_reel PASSED [ 61%]
|
||||
tests/unit/test_darwin_engine_comments.py::test_has_comments_true_organic PASSED [ 62%]
|
||||
tests/unit/test_darwin_engine_comments.py::test_has_comments_zero_reel PASSED [ 62%]
|
||||
tests/unit/test_darwin_engine_comments.py::test_has_comments_regex_cases PASSED [ 63%]
|
||||
tests/unit/test_dopamine_engine.py::test_dopamine_engine_wants_to_change_feed PASSED [ 63%]
|
||||
tests/unit/test_dopamine_engine.py::test_dopamine_engine_reset_session_clears_boredom PASSED [ 64%]
|
||||
tests/unit/test_dopamine_engine.py::test_dopamine_engine_wants_to_doomscroll PASSED [ 65%]
|
||||
tests/unit/test_dopamine_loop.py::test_feed_switch_resets_boredom PASSED [ 65%]
|
||||
tests/unit/test_dopamine_loop.py::test_session_limit_terminates_session PASSED [ 66%]
|
||||
tests/unit/test_feed_loop_continuation.py::TestFeedLoopContinuation::test_stories_complete_returns_feed_exhausted PASSED [ 66%]
|
||||
tests/unit/test_feed_loop_continuation.py::TestFeedLoopContinuation::test_main_loop_handles_feed_exhausted PASSED [ 67%]
|
||||
tests/unit/test_goap_graph_routing.py::TestGoapGraphRouting::test_planner_routes_to_profile_first_for_following_list PASSED [ 67%]
|
||||
tests/unit/test_goap_graph_routing.py::TestGoapGraphRouting::test_planner_returns_final_action_on_intermediate_screen PASSED [ 68%]
|
||||
tests/unit/test_goap_graph_routing.py::TestGoapGraphRouting::test_planner_detects_goal_already_achieved PASSED [ 68%]
|
||||
tests/unit/test_goap_graph_routing.py::TestGoapGraphRouting::test_planner_routes_explore_to_following_list PASSED [ 69%]
|
||||
tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_first_call_returns_topmost_leftmost FAILED [ 69%]
|
||||
tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_retry_skips_failed_position FAILED [ 70%]
|
||||
tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_skip_multiple_positions FAILED [ 70%]
|
||||
tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_all_positions_skipped_returns_none FAILED [ 71%]
|
||||
tests/unit/test_is_ad_substring.py::test_is_ad_false_positive_abroad PASSED [ 72%]
|
||||
tests/unit/test_is_ad_substring.py::test_is_ad_true_positive PASSED [ 72%]
|
||||
tests/unit/test_is_ad_substring.py::test_is_ad_true_positive_ad_word PASSED [ 73%]
|
||||
tests/unit/test_nav_intent_classification.py::TestNavIntentClassification::test_dm_intent_is_classified_as_nav_intent PASSED [ 73%]
|
||||
tests/unit/test_nav_intent_classification.py::TestNavIntentClassification::test_inbox_intent_is_classified_as_nav_intent PASSED [ 74%]
|
||||
tests/unit/test_nav_intent_classification.py::TestNavIntentClassification::test_notification_intent_is_classified_as_nav_intent PASSED [ 74%]
|
||||
tests/unit/test_nav_intent_classification.py::TestNavIntentClassification::test_regular_post_intent_still_blocked_in_nav_zone PASSED [ 75%]
|
||||
tests/unit/test_screen_identity_profile.py::test_screen_identity_own_profile_vs_other_profile PASSED [ 75%]
|
||||
tests/unit/test_screen_identity_profile.py::test_screen_identity_other_profile_vs_own_profile PASSED [ 76%]
|
||||
tests/unit/test_screen_topology.py::TestScreenTopologyRouting::test_route_home_to_following_list PASSED [ 76%]
|
||||
tests/unit/test_screen_topology.py::TestScreenTopologyRouting::test_route_already_there PASSED [ 77%]
|
||||
tests/unit/test_screen_topology.py::TestScreenTopologyRouting::test_route_single_hop PASSED [ 77%]
|
||||
tests/unit/test_screen_topology.py::TestScreenTopologyRouting::test_route_reverse_direction PASSED [ 78%]
|
||||
tests/unit/test_screen_topology.py::TestScreenTopologyRouting::test_no_route_from_unreachable PASSED [ 79%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_following_list_goal PASSED [ 79%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_followers_list_goal PASSED [ 80%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_profile_goal PASSED [ 80%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_home_feed_goal PASSED [ 81%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_explore_goal PASSED [ 81%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_messages_goal PASSED [ 82%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_interaction_goal_returns_none PASSED [ 82%]
|
||||
tests/unit/test_screen_topology.py::TestGoalToTargetScreen::test_unknown_goal_returns_none PASSED [ 83%]
|
||||
tests/unit/test_screen_topology.py::TestGetTransitions::test_home_feed_has_profile_tab PASSED [ 83%]
|
||||
tests/unit/test_screen_topology.py::TestGetTransitions::test_own_profile_has_following_list PASSED [ 84%]
|
||||
tests/unit/test_screen_topology.py::TestGetTransitions::test_unknown_screen_returns_empty PASSED [ 84%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameMap::test_following_list_maps PASSED [ 85%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameMap::test_home_feed_maps PASSED [ 86%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameMap::test_stories_feed_maps_to_home PASSED [ 86%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameMap::test_search_feed_maps_to_explore PASSED [ 87%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameToGoal::test_following_list PASSED [ 87%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameToGoal::test_home_feed PASSED [ 88%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameToGoal::test_explore_feed PASSED [ 88%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameToGoal::test_stories_feed PASSED [ 89%]
|
||||
tests/unit/test_screen_topology.py::TestScreenNameToGoal::test_unknown_target PASSED [ 89%]
|
||||
tests/unit/test_screen_topology.py::TestExpectedScreenForAction::test_tap_profile_tab_from_home PASSED [ 90%]
|
||||
tests/unit/test_screen_topology.py::TestExpectedScreenForAction::test_tap_following_list_from_profile PASSED [ 90%]
|
||||
tests/unit/test_screen_topology.py::TestExpectedScreenForAction::test_press_back_from_follow_list PASSED [ 91%]
|
||||
tests/unit/test_screen_topology.py::TestExpectedScreenForAction::test_unknown_action_returns_none PASSED [ 91%]
|
||||
tests/unit/test_screen_topology.py::TestExpectedScreenForAction::test_action_not_available_on_screen PASSED [ 92%]
|
||||
tests/unit/test_screen_topology.py::TestIsStructuralAction::test_tap_profile_tab_is_structural PASSED [ 93%]
|
||||
tests/unit/test_screen_topology.py::TestIsStructuralAction::test_tap_following_list_is_structural PASSED [ 93%]
|
||||
tests/unit/test_screen_topology.py::TestIsStructuralAction::test_random_action_is_not_structural PASSED [ 94%]
|
||||
tests/unit/test_screen_topology.py::TestIsStructuralAction::test_action_on_wrong_screen_is_not_structural PASSED [ 94%]
|
||||
tests/unit/test_session_limits_evaluation.py::test_global_session_limit_evaluation ERROR [ 95%]
|
||||
tests/unit/test_structural_guard.py::test_structural_guard_rejects_own_story_for_post_username PASSED [ 95%]
|
||||
tests/unit/test_structural_guard.py::test_structural_guard_accepts_actual_post_username PASSED [ 96%]
|
||||
tests/unit/test_structural_guard.py::test_structural_guard_rejects_own_username_story PASSED [ 96%]
|
||||
tests/unit/test_structural_guard.py::test_structural_reels_first_grid_item_y_coords PASSED [ 97%]
|
||||
tests/unit/test_telepathic_container_filtering.py::test_media_intent_rejects_grid_containers PASSED [ 97%]
|
||||
tests/unit/test_verify_success_reels.py::TestVerifySuccessGridReels::test_reel_view_accepted_as_valid_grid_result PASSED [ 98%]
|
||||
tests/unit/test_verify_success_reels.py::TestVerifySuccessGridReels::test_normal_feed_post_still_accepted PASSED [ 98%]
|
||||
tests/unit/test_verify_success_reels.py::TestVerifySuccessGridReels::test_explore_grid_still_visible_is_failure PASSED [ 99%]
|
||||
tests/unit/test_verify_success_reels.py::TestVerifySuccessGridReels::test_profile_grid_reel_accepted PASSED [100%]
|
||||
|
||||
==================================== ERRORS ====================================
|
||||
______ ERROR at setup of TestSAEPerception.test_perceive_normal_instagram ______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_____ ERROR at setup of TestSAEPerception.test_perceive_foreign_app_google _____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_____ ERROR at setup of TestSAEPerception.test_perceive_notification_shade _____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
__ ERROR at setup of TestSAEPerception.test_perceive_system_permission_dialog __
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___ ERROR at setup of TestSAEPerception.test_perceive_instagram_survey_modal ___
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAEPerception.test_perceive_unknown_modal_interstitial _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_______ ERROR at setup of TestSAEPerception.test_perceive_action_blocked _______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_________ ERROR at setup of TestSAEPerception.test_perceive_empty_dump _________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_________ ERROR at setup of TestSAEPerception.test_perceive_none_dump __________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAEPerception.test_perceive_passive_scaffold_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_home_feed_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_explore_grid_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_other_profile_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_post_detail_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_profile_tagged_tab_as_normal _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_survey_modal_as_obstacle _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_ ERROR at setup of TestSAERealFixturePerception.test_perceive_mystery_interstitial_as_obstacle _
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___ ERROR at setup of test_goap_planner_avoids_infinite_loop_on_masked_edge ____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
____ ERROR at setup of test_screen_topology_find_route_avoids_blocked_edges ____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___ ERROR at setup of test_telepathic_engine_finds_following_node_on_profile ___
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
______ ERROR at setup of test_following_vs_followers_are_both_candidates _______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___________ ERROR at setup of test_vlm_prompt_humanizes_content_desc ___________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_______ ERROR at setup of test_live_vlm_selects_following_not_followers ________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_____________ ERROR at setup of test_reel_like_button_not_caption ______________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
______ ERROR at setup of test_reel_follow_button_returns_none_when_absent ______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___________ ERROR at setup of test_reel_post_author_selects_username ___________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
___________ ERROR at setup of test_reel_dedup_preserves_like_button ____________
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
____ ERROR at setup of test_reel_caption_with_like_word_is_not_like_button _____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
__ ERROR at setup of test_intent_resolver_profile_tab_rejects_author_profile ___
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_____ ERROR at setup of test_intent_resolver_profile_tab_selects_real_tab ______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
_____ ERROR at setup of test_visual_discovery_creates_annotated_screenshot _____
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
______ ERROR at setup of test_visual_discovery_finds_following_by_seeing _______
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
__ ERROR at setup of test_resolve_uses_visual_discovery_when_device_available __
|
||||
/Users/marcmintel/.local/lib/python3.11/site-packages/_pytest/config/__init__.py:1694: in getoption
|
||||
val = getattr(self.option, name)
|
||||
E AttributeError: 'Namespace' object has no attribute '--live'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
tests/e2e/conftest.py:195: in mock_all_delays
|
||||
if request.config.getoption("--live"):
|
||||
E ValueError: no option named '--live'
|
||||
____________ ERROR at setup of test_global_session_limit_evaluation ____________
|
||||
file /Volumes/Alpha SSD/Coding/bot/tests/unit/test_session_limits_evaluation.py, line 1
|
||||
def test_global_session_limit_evaluation(mock_logger):
|
||||
E fixture 'mock_logger' not found
|
||||
> available fixtures: _session_faker, anyio_backend, anyio_backend_name, anyio_backend_options, benchmark, benchmark_weave, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, class_mocker, cov, datadir, doctest_namespace, extra, extras, faker, httpserver, httpserver_ipv4, httpserver_ipv6, httpserver_listen_address, httpserver_ssl_context, include_metadata_in_junit_xml, json_metadata, lazy_datadir, lazy_shared_datadir, make_httpserver, make_httpserver_ipv4, make_httpserver_ipv6, metadata, mocker, module_mocker, monkeypatch, no_cover, original_datadir, package_mocker, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, session_mocker, shared_datadir, snapshot, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, worker_id
|
||||
> use 'pytest --fixtures [testpath]' for help on them.
|
||||
|
||||
/Volumes/Alpha SSD/Coding/bot/tests/unit/test_session_limits_evaluation.py:1
|
||||
=================================== FAILURES ===================================
|
||||
______________ TestTelepathicGuards.test_strict_story_ring_guard _______________
|
||||
tests/anomalies/test_telepathic_guards.py:23: in test_strict_story_ring_guard
|
||||
assert self.engine._structural_sanity_check(invalid_story, intent, screen_height) is False
|
||||
E AssertionError: assert True is False
|
||||
E + where True = _structural_sanity_check({'area': 100, 'resource_id': 'row_feed_profile_header', 'y': 800}, 'tap story ring avatar', 2400)
|
||||
E + where _structural_sanity_check = <GramAddict.core.telepathic_engine.TelepathicEngine object at 0x137e50e50>._structural_sanity_check
|
||||
E + where <GramAddict.core.telepathic_engine.TelepathicEngine object at 0x137e50e50> = <tests.anomalies.test_telepathic_guards.TestTelepathicGuards object at 0x137cc5190>.engine
|
||||
________________ TestTelepathicGuards.test_strict_button_guard _________________
|
||||
tests/anomalies/test_telepathic_guards.py:45: in test_strict_button_guard
|
||||
assert self.engine._structural_sanity_check(invalid_prof, intent, screen_height) is False
|
||||
E assert True is False
|
||||
E + where True = _structural_sanity_check({'area': 100, 'resource_id': 'username', 'semantic_string': "Go to cayleighanddavid's profile", 'y': 1000}, 'Heart like button for comment', 2400)
|
||||
E + where _structural_sanity_check = <GramAddict.core.telepathic_engine.TelepathicEngine object at 0x138184dd0>._structural_sanity_check
|
||||
E + where <GramAddict.core.telepathic_engine.TelepathicEngine object at 0x138184dd0> = <tests.anomalies.test_telepathic_guards.TestTelepathicGuards object at 0x137cc58d0>.engine
|
||||
__________________________ test_keyword_nav_threshold __________________________
|
||||
tests/integration/test_telepathic_hardening.py:37: in test_keyword_nav_threshold
|
||||
res = engine._keyword_match_score("tap messages tab", [reels_node])
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_keyword_match_score'
|
||||
__________________________ test_direct_tab_fast_path ___________________________
|
||||
tests/integration/test_telepathic_hardening.py:57: in test_direct_tab_fast_path
|
||||
res = engine._core_navigation_fast_path("tap messages tab", [direct_node])
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_core_navigation_fast_path'
|
||||
___________________ test_keyword_fast_path_no_feed_pollution ___________________
|
||||
tests/integration/test_telepathic_keyword.py:30: in test_keyword_fast_path_no_feed_pollution
|
||||
result = engine._keyword_match_score("tap home tab", nodes)
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_keyword_match_score'
|
||||
_______ TestPositionRejection.test_repro_following_button_rejection_fix ________
|
||||
tests/repro_reports/test_repro_position_rejection.py:34: in test_repro_following_button_rejection_fix
|
||||
self.assertTrue(passed_keyword, "Following button should be allowed for following intent")
|
||||
E AssertionError: False is not true : Following button should be allowed for following intent
|
||||
----------------------------- Captured stdout call -----------------------------
|
||||
|
||||
[DEBUG] Intent: 'tap following list', Passed: False
|
||||
|
||||
[DEBUG] Intent: 'some other intent', Passed: False
|
||||
___________ TestReproReelsTabHallucination.test_reels_tab_selection ____________
|
||||
tests/repro_reports/test_repro_reels_tab_hallucination.py:49: in test_reels_tab_selection
|
||||
self.assertIn("clips tab", result["semantic"].lower(), "Should select the clips_tab")
|
||||
E KeyError: 'semantic'
|
||||
----------------------------- Captured stdout call -----------------------------
|
||||
FIND_BEST_NODE CALLED
|
||||
Target selected: None at (324, 2298)
|
||||
____________________ test_intent_resolver_finds_bottom_tab _____________________
|
||||
tests/unit/perception/test_intent_resolver.py:16: in test_intent_resolver_finds_bottom_tab
|
||||
assert best_match == bottom_tab
|
||||
E AssertionError: assert None == SpatialNode(bounds=(0, 2200, 100, 2300), node_id='', class_name='', text='', content_desc='Explore Tab', resource_id='', clickable=True, scrollable=False, children=[], parent=None)
|
||||
_______ TestGridRetryDiversity.test_first_call_returns_topmost_leftmost ________
|
||||
tests/unit/test_grid_retry_diversity.py:84: in test_first_call_returns_topmost_leftmost
|
||||
result = self.engine._grid_fast_path("first image in explore grid", nodes)
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_grid_fast_path'
|
||||
___________ TestGridRetryDiversity.test_retry_skips_failed_position ____________
|
||||
tests/unit/test_grid_retry_diversity.py:92: in test_retry_skips_failed_position
|
||||
result = self.engine._grid_fast_path("first image in explore grid", nodes, skip_positions={(178, 558)})
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_grid_fast_path'
|
||||
_____________ TestGridRetryDiversity.test_skip_multiple_positions ______________
|
||||
tests/unit/test_grid_retry_diversity.py:99: in test_skip_multiple_positions
|
||||
result = self.engine._grid_fast_path(
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_grid_fast_path'
|
||||
________ TestGridRetryDiversity.test_all_positions_skipped_returns_none ________
|
||||
tests/unit/test_grid_retry_diversity.py:109: in test_all_positions_skipped_returns_none
|
||||
result = self.engine._grid_fast_path("first image in explore grid", nodes, skip_positions=all_positions)
|
||||
E AttributeError: 'TelepathicEngine' object has no attribute '_grid_fast_path'
|
||||
=============================== warnings summary ===============================
|
||||
../../../../Users/marcmintel/.pyenv/versions/3.11.9/lib/python3.11/site-packages/requests/__init__.py:109
|
||||
/Users/marcmintel/.pyenv/versions/3.11.9/lib/python3.11/site-packages/requests/__init__.py:109: RequestsDependencyWarning: urllib3 (2.4.0) or chardet (7.4.3)/charset_normalizer (3.4.2) doesn't match a supported version!
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
=========================== short test summary info ============================
|
||||
FAILED tests/anomalies/test_telepathic_guards.py::TestTelepathicGuards::test_strict_story_ring_guard
|
||||
FAILED tests/anomalies/test_telepathic_guards.py::TestTelepathicGuards::test_strict_button_guard
|
||||
FAILED tests/integration/test_telepathic_hardening.py::test_keyword_nav_threshold
|
||||
FAILED tests/integration/test_telepathic_hardening.py::test_direct_tab_fast_path
|
||||
FAILED tests/integration/test_telepathic_keyword.py::test_keyword_fast_path_no_feed_pollution
|
||||
FAILED tests/repro_reports/test_repro_position_rejection.py::TestPositionRejection::test_repro_following_button_rejection_fix
|
||||
FAILED tests/repro_reports/test_repro_reels_tab_hallucination.py::TestReproReelsTabHallucination::test_reels_tab_selection
|
||||
FAILED tests/unit/perception/test_intent_resolver.py::test_intent_resolver_finds_bottom_tab
|
||||
FAILED tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_first_call_returns_topmost_leftmost
|
||||
FAILED tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_retry_skips_failed_position
|
||||
FAILED tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_skip_multiple_positions
|
||||
FAILED tests/unit/test_grid_retry_diversity.py::TestGridRetryDiversity::test_all_positions_skipped_returns_none
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_normal_instagram
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_foreign_app_google
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_notification_shade
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_system_permission_dialog
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_instagram_survey_modal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_unknown_modal_interstitial
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_action_blocked
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_empty_dump
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_none_dump
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAEPerception::test_perceive_passive_scaffold_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_home_feed_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_explore_grid_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_other_profile_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_post_detail_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_profile_tagged_tab_as_normal
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_survey_modal_as_obstacle
|
||||
ERROR tests/e2e/test_engine_perception.py::TestSAERealFixturePerception::test_perceive_mystery_interstitial_as_obstacle
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_goap_planner_avoids_infinite_loop_on_masked_edge
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_screen_topology_find_route_avoids_blocked_edges
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_telepathic_engine_finds_following_node_on_profile
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_following_vs_followers_are_both_candidates
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_vlm_prompt_humanizes_content_desc
|
||||
ERROR tests/e2e/test_goap_loop_prevention.py::test_live_vlm_selects_following_not_followers
|
||||
ERROR tests/e2e/test_reel_interactions.py::test_reel_like_button_not_caption
|
||||
ERROR tests/e2e/test_reel_interactions.py::test_reel_follow_button_returns_none_when_absent
|
||||
ERROR tests/e2e/test_reel_interactions.py::test_reel_post_author_selects_username
|
||||
ERROR tests/e2e/test_reel_interactions.py::test_reel_dedup_preserves_like_button
|
||||
ERROR tests/e2e/test_reel_interactions.py::test_reel_caption_with_like_word_is_not_like_button
|
||||
ERROR tests/e2e/test_reel_navigation_guards.py::test_intent_resolver_profile_tab_rejects_author_profile
|
||||
ERROR tests/e2e/test_reel_navigation_guards.py::test_intent_resolver_profile_tab_selects_real_tab
|
||||
ERROR tests/e2e/test_visual_intent_resolver.py::test_visual_discovery_creates_annotated_screenshot
|
||||
ERROR tests/e2e/test_visual_intent_resolver.py::test_visual_discovery_finds_following_by_seeing
|
||||
ERROR tests/e2e/test_visual_intent_resolver.py::test_resolve_uses_visual_discovery_when_device_available
|
||||
ERROR tests/unit/test_session_limits_evaluation.py::test_global_session_limit_evaluation
|
||||
======= 12 failed, 135 passed, 5 skipped, 1 warning, 34 errors in 19.92s =======
|
||||
BIN
tests/.DS_Store
vendored
Normal file
BIN
tests/.DS_Store
vendored
Normal file
Binary file not shown.
BIN
tests/__pycache__/__init__.cpython-311.pyc
Normal file
BIN
tests/__pycache__/__init__.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/conftest.cpython-311-pytest-8.3.5.pyc
Normal file
BIN
tests/__pycache__/conftest.cpython-311-pytest-8.3.5.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
tests/anomalies/__pycache__/__init__.cpython-311.pyc
Normal file
BIN
tests/anomalies/__pycache__/__init__.cpython-311.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,198 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Force mock qdrant_client before importing any core modules that depend on it
|
||||
from GramAddict.core.bot_flow import _extract_post_content, _run_zero_latency_feed_loop
|
||||
|
||||
|
||||
class TestBotFlowEdgeCases:
|
||||
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
|
||||
def test_extract_post_content_edge_cases(self, mock_get_telepathic):
|
||||
mock_engine = MagicMock()
|
||||
mock_get_telepathic.return_value = mock_engine
|
||||
|
||||
# 1. Empty string / Invalid XML should not crash (mock finds nothing)
|
||||
mock_engine.find_best_node.return_value = None
|
||||
res = _extract_post_content("")
|
||||
assert res.get("username") == ""
|
||||
assert res.get("description") == ""
|
||||
|
||||
# 2. Extract when only username exists
|
||||
# Side effect: first call (author) returns node, second (media) returns None
|
||||
mock_engine.find_best_node.side_effect = [{"original_attribs": {"text": "just_user"}}, None]
|
||||
res = _extract_post_content("<xml/>")
|
||||
assert res.get("username") == "just_user"
|
||||
assert res.get("description") == ""
|
||||
|
||||
# 3. Extract description
|
||||
mock_engine.find_best_node.side_effect = [None, {"original_attribs": {"desc": "🔥🔥🔥🔥🔥🔥🔥🔥🔥🔥🔥"}}]
|
||||
res = _extract_post_content("<xml/>")
|
||||
assert res.get("description") == "🔥🔥🔥🔥🔥🔥🔥🔥🔥🔥🔥"
|
||||
|
||||
# 4. Another valid description tag
|
||||
mock_engine.find_best_node.side_effect = [
|
||||
None,
|
||||
{"original_attribs": {"desc": "some desc with more than 10 chars limits"}},
|
||||
]
|
||||
res = _extract_post_content("<xml/>")
|
||||
assert res.get("description") == "some desc with more than 10 chars limits"
|
||||
|
||||
@patch("GramAddict.core.bot_flow.random.random", return_value=0.5)
|
||||
@patch("GramAddict.core.bot_flow.random.uniform", return_value=1.5)
|
||||
@patch("GramAddict.core.bot_flow.sleep")
|
||||
@patch("GramAddict.core.bot_flow._humanized_scroll")
|
||||
@patch("GramAddict.core.utils.is_ad")
|
||||
@patch("GramAddict.core.bot_flow._align_active_post")
|
||||
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
|
||||
def test_zero_node_recovery(
|
||||
self, mock_get_telepathic, mock_align, mock_ad, mock_scroll, mock_sleep, mock_uniform, mock_random
|
||||
):
|
||||
# Tests the explicit Zero-Node Recovery added previously
|
||||
device = MagicMock()
|
||||
zero_engine = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = MagicMock()
|
||||
session_state = MagicMock()
|
||||
|
||||
mock_ad.return_value = False
|
||||
mock_align.return_value = False
|
||||
|
||||
cognitive_stack = {
|
||||
"dopamine": MagicMock(),
|
||||
"darwin": MagicMock(),
|
||||
"resonance": MagicMock(),
|
||||
"active_inference": MagicMock(),
|
||||
"growth_brain": MagicMock(),
|
||||
"swarm": MagicMock(),
|
||||
}
|
||||
|
||||
# Dopamine breaks loop after 1st iteration
|
||||
cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
|
||||
cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
|
||||
cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
|
||||
|
||||
# Fake extreme limits => doesn't break limits
|
||||
session_state.check_limit.return_value = [False] * 10
|
||||
|
||||
# Telepathic Engine returns ZERO nodes on extract
|
||||
mock_engine = MagicMock()
|
||||
mock_engine._extract_semantic_nodes.return_value = []
|
||||
mock_get_telepathic.return_value = mock_engine
|
||||
|
||||
device.dump_hierarchy.return_value = "<xml></xml>"
|
||||
|
||||
# Execute the main loop
|
||||
_run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session_state, "HomeFeed", cognitive_stack)
|
||||
|
||||
# It should trigger _humanized_scroll
|
||||
assert mock_scroll.call_count >= 1
|
||||
|
||||
@patch("GramAddict.core.bot_flow.random.random", return_value=0.5)
|
||||
@patch("GramAddict.core.bot_flow.random.uniform", return_value=1.5)
|
||||
@patch("GramAddict.core.bot_flow.sleep")
|
||||
@patch("GramAddict.core.bot_flow._humanized_scroll")
|
||||
@patch("GramAddict.core.bot_flow._extract_post_content")
|
||||
@patch("GramAddict.core.utils.is_ad")
|
||||
@patch("GramAddict.core.bot_flow._align_active_post")
|
||||
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
|
||||
def test_content_extraction_failed_recovery(
|
||||
self,
|
||||
mock_get_telepathic,
|
||||
mock_align,
|
||||
mock_ad,
|
||||
mock_extract,
|
||||
mock_scroll,
|
||||
mock_sleep,
|
||||
mock_uniform,
|
||||
mock_random,
|
||||
):
|
||||
device = MagicMock()
|
||||
zero_engine = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = MagicMock()
|
||||
session_state = MagicMock()
|
||||
|
||||
mock_ad.return_value = False
|
||||
mock_align.return_value = False
|
||||
|
||||
cognitive_stack = {"dopamine": MagicMock(), "darwin": MagicMock()}
|
||||
# break after 1 loop
|
||||
cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
|
||||
cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
|
||||
cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
|
||||
session_state.check_limit.return_value = [False] * 10
|
||||
|
||||
# Ensure it HAS feed markers
|
||||
device.dump_hierarchy.return_value = "<xml>row_feed_photo_profile_name</xml>"
|
||||
|
||||
# Ensure interactive_nodes is NOT zero
|
||||
mock_engine = MagicMock()
|
||||
mock_engine._extract_semantic_nodes.return_value = [{"x": 10}]
|
||||
mock_get_telepathic.return_value = mock_engine
|
||||
|
||||
# Make the extraction fail
|
||||
mock_extract.return_value = {"username": "", "description": ""}
|
||||
|
||||
_run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session_state, "HomeFeed", cognitive_stack)
|
||||
|
||||
# Should call mock_scroll (Graceful degradation)
|
||||
mock_scroll.assert_called_once()
|
||||
|
||||
@patch("GramAddict.core.bot_flow.sleep")
|
||||
@patch("GramAddict.core.bot_flow._humanized_scroll")
|
||||
@patch("GramAddict.core.utils.is_ad")
|
||||
@patch("GramAddict.core.bot_flow._align_active_post")
|
||||
@patch("GramAddict.core.bot_flow._extract_post_content")
|
||||
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
|
||||
@patch("GramAddict.core.llm_provider.query_llm")
|
||||
def test_llm_timeout_handled_smoothly(
|
||||
self, mock_query_llm, mock_get_telepathic, mock_extract, mock_align, mock_ad, mock_scroll, mock_sleep
|
||||
):
|
||||
"""
|
||||
TDD Test: Verifies that if qwen3.5:latest times out during comment generation
|
||||
(simulated by query_llm returning None after circuit breaker), the bot_flow
|
||||
catches the empty response and continues gracefully without crashing.
|
||||
"""
|
||||
device = MagicMock()
|
||||
zero_engine = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = MagicMock()
|
||||
session_state = MagicMock()
|
||||
|
||||
mock_ad.return_value = False
|
||||
mock_align.return_value = False
|
||||
|
||||
# Make the LLM generation completely timeout and return None
|
||||
mock_query_llm.return_value = None
|
||||
|
||||
cognitive_stack = {"dopamine": MagicMock(), "darwin": MagicMock(), "resonance": MagicMock()}
|
||||
# break after 1 loop
|
||||
cognitive_stack["dopamine"].is_app_session_over.side_effect = [False, True]
|
||||
cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
|
||||
cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
|
||||
|
||||
# Emulate that dopamine WANTS to comment
|
||||
cognitive_stack["dopamine"].get_action_desires.return_value = {"comment": True, "like": False}
|
||||
|
||||
# Avoid MagicMock comparison errors in Resonance Engine
|
||||
cognitive_stack["resonance"].calculate_resonance.return_value = 0.8
|
||||
|
||||
session_state.check_limit.return_value = [False] * 10
|
||||
|
||||
device.dump_hierarchy.return_value = "<xml>row_feed_photo_profile_name</xml>"
|
||||
|
||||
mock_engine = MagicMock()
|
||||
mock_engine._extract_semantic_nodes.return_value = [{"x": 10}]
|
||||
mock_get_telepathic.return_value = mock_engine
|
||||
|
||||
# Valid post content so it proceeds to comment generation
|
||||
mock_extract.return_value = {"username": "test_user", "description": "a long enough description"}
|
||||
|
||||
# Run feed loop - MUST NOT CRASH
|
||||
try:
|
||||
_run_zero_latency_feed_loop(
|
||||
device, zero_engine, nav_graph, configs, session_state, "HomeFeed", cognitive_stack
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.fail(f"Feed loop crashed on LLM timeout with {e}")
|
||||
@@ -1,69 +0,0 @@
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Mock directory setup
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
FIXTURE_DIR = os.path.join(ROOT_DIR, "fixtures")
|
||||
|
||||
|
||||
class ConfigMock:
|
||||
def __init__(self):
|
||||
self.args = MagicMock()
|
||||
self.args.app_id = "com.instagram.android"
|
||||
|
||||
|
||||
def test_fsd_handles_persistent_survey_modal():
|
||||
"""
|
||||
Simulates a case where the bot gets stuck on a survey modal.
|
||||
The FSD (Full Self Driving) anomaly handler should trigger,
|
||||
detect that 'Back' didn't work, and engage TelepathicEngine
|
||||
to find and tap the 'Not Now' or 'Dismiss' button.
|
||||
"""
|
||||
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
|
||||
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device._get_current_app.return_value = "com.instagram.android"
|
||||
configs = ConfigMock()
|
||||
|
||||
# Mock the TelepathicEngine singleton behavior entirely
|
||||
mock_telepathic = MagicMock()
|
||||
mock_telepathic.find_best_node.return_value = {"x": 500, "y": 1400, "semantic": "Not Now"}
|
||||
|
||||
dopamine = MagicMock()
|
||||
dopamine.is_app_session_over.side_effect = [False, False, True] # Run twice, then exit
|
||||
dopamine.wants_to_change_feed.return_value = False
|
||||
dopamine.wants_to_doomscroll.return_value = False
|
||||
|
||||
ai = MagicMock()
|
||||
ai.get_sleep_modifier.return_value = 1.0
|
||||
|
||||
cognitive_stack = {
|
||||
"dopamine": dopamine,
|
||||
"growth_brain": None,
|
||||
"active_inference": ai,
|
||||
}
|
||||
|
||||
# Load the mock survey modal UI
|
||||
xml_path = os.path.join(FIXTURE_DIR, "survey_modal.xml")
|
||||
with open(xml_path, "r") as f:
|
||||
alien_xml = f.read()
|
||||
device.dump_hierarchy.return_value = alien_xml
|
||||
|
||||
with (
|
||||
patch("GramAddict.core.bot_flow.sleep"),
|
||||
patch("GramAddict.core.behaviors.obstacle_guard.sleep"),
|
||||
patch("GramAddict.core.bot_flow._humanized_scroll"),
|
||||
patch("GramAddict.core.behaviors.obstacle_guard.TelepathicEngine.get_instance", return_value=mock_telepathic),
|
||||
patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance", return_value=mock_telepathic),
|
||||
):
|
||||
from GramAddict.core.behaviors import PluginRegistry
|
||||
from GramAddict.core.behaviors.obstacle_guard import ObstacleGuardPlugin
|
||||
|
||||
PluginRegistry.get_instance().register(ObstacleGuardPlugin())
|
||||
|
||||
_run_zero_latency_feed_loop(device, None, MagicMock(), configs, MagicMock(), "HomeFeed", cognitive_stack)
|
||||
|
||||
# VERIFICATION:
|
||||
assert mock_telepathic.find_best_node.called
|
||||
assert device.click.called
|
||||
@@ -1,78 +0,0 @@
|
||||
"""
|
||||
TDD Tests for Zero-Hardcode Screen Classification and Situational Awareness
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")))
|
||||
|
||||
from GramAddict.core.goap import ScreenIdentity, ScreenType
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_screen_memory():
|
||||
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB") as mock_db:
|
||||
instance = mock_db.return_value
|
||||
instance.is_connected = True
|
||||
yield instance
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_query_llm():
|
||||
with patch("GramAddict.core.llm_provider.query_llm") as mock_llm:
|
||||
yield mock_llm
|
||||
|
||||
|
||||
def test_classify_screen_uses_memory(mock_screen_memory, mock_query_llm):
|
||||
"""
|
||||
Test that _classify_screen FIRST tries to hit the ScreenMemoryDB.
|
||||
"""
|
||||
si = ScreenIdentity("testbot")
|
||||
|
||||
# Mock that memory ALREADY knows this screen
|
||||
mock_screen_memory.get_screen_type.return_value = ScreenType.MODAL.name
|
||||
|
||||
# We pass random strings that would previously fail or hit hardcoded checks
|
||||
res = si._classify_screen(
|
||||
ids=set(),
|
||||
descs=[],
|
||||
texts=["totally ambiguous text"],
|
||||
selected_tab=None,
|
||||
desc_lower="",
|
||||
text_lower="",
|
||||
ids_str="random_id",
|
||||
signature="MOCK_SIGNATURE",
|
||||
)
|
||||
|
||||
assert res == ScreenType.MODAL
|
||||
mock_screen_memory.get_screen_type.assert_called_once_with("MOCK_SIGNATURE", similarity_threshold=0.92)
|
||||
# Should not fall back to LLM if memory hits
|
||||
mock_query_llm.assert_not_called()
|
||||
|
||||
|
||||
def test_classify_screen_uses_llm_fallback_and_learns(mock_screen_memory, mock_query_llm):
|
||||
"""
|
||||
Test that if memory misses, it uses LLM fallback and caches the result.
|
||||
"""
|
||||
si = ScreenIdentity("testbot")
|
||||
mock_screen_memory.get_screen_type.return_value = None
|
||||
mock_query_llm.return_value = {"response": "HOME_FEED"}
|
||||
|
||||
res = si._classify_screen(
|
||||
ids={"random"},
|
||||
descs=[],
|
||||
texts=[],
|
||||
selected_tab=None,
|
||||
desc_lower="",
|
||||
text_lower="",
|
||||
ids_str="random",
|
||||
signature="MOCK_SIGNATURE_2",
|
||||
)
|
||||
|
||||
assert res == ScreenType.HOME_FEED
|
||||
mock_query_llm.assert_called_once()
|
||||
mock_screen_memory.store_screen.assert_called_once_with("MOCK_SIGNATURE_2", "HOME_FEED")
|
||||
@@ -1,190 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop, _wait_for_post_loaded
|
||||
from GramAddict.core.perception.feed_analysis import FEED_MARKERS
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DUMPS = {
|
||||
"organic": os.path.join(ROOT_DIR, "fixtures", "organic_post.xml"),
|
||||
"explore": os.path.join(ROOT_DIR, "fixtures", "explore_feed_dump.xml"),
|
||||
}
|
||||
FIXTURE_DIR = os.path.join(ROOT_DIR, "fixtures")
|
||||
|
||||
|
||||
def mutate_xml_to_foreign(xml_content: str) -> str:
|
||||
"""Removes meaningful text content to simulate a language failure or empty state."""
|
||||
import re
|
||||
|
||||
# Strip text and content-desc
|
||||
xml = re.sub(r'text="[^"]*"', 'text=""', xml_content)
|
||||
xml = re.sub(r'content-desc="[^"]*"', 'content-desc=""', xml)
|
||||
return xml
|
||||
|
||||
|
||||
def mutate_xml_remove_feed_markers(xml_content: str) -> str:
|
||||
"""Removes all feed markers to simulate a grid view or random popup."""
|
||||
xml = xml_content
|
||||
for marker in FEED_MARKERS:
|
||||
xml = xml.replace(marker, "some_random_id")
|
||||
return xml
|
||||
|
||||
|
||||
class ConfigMock:
|
||||
def __init__(self):
|
||||
self.args = MagicMock()
|
||||
self.args.interact_percentage = 0
|
||||
self.args.comment_percentage = 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_dumps():
|
||||
dumps = {}
|
||||
with open(DUMPS["organic"], "r") as f:
|
||||
dumps["post"] = f.read()
|
||||
# Fake explore grid that lacks ALL feed markers
|
||||
dumps["grid"] = (
|
||||
'<?xml version="1.0"?><hierarchy><node resource-id="com.instagram.android:id/explore_grid_container" /></hierarchy>'
|
||||
)
|
||||
return dumps
|
||||
|
||||
|
||||
def test_slow_loading_post_recovery(test_dumps):
|
||||
"""
|
||||
Test that _wait_for_post_loaded correctly handles a delay where the
|
||||
first few dumps are grids, and only later it becomes a post.
|
||||
"""
|
||||
device = MagicMock()
|
||||
# Simulate: Grid -> Grid -> Error -> Post
|
||||
device.dump_hierarchy.side_effect = [
|
||||
test_dumps["grid"],
|
||||
test_dumps["grid"],
|
||||
Exception("uiautomator2 temp failure"),
|
||||
test_dumps["post"],
|
||||
]
|
||||
|
||||
# We patch sleep to make the test super fast
|
||||
with patch("GramAddict.core.bot_flow.sleep", return_value=None):
|
||||
time.time()
|
||||
success = _wait_for_post_loaded(device, timeout=5)
|
||||
# Should return true when it hits the 4th element
|
||||
assert success is True
|
||||
assert device.dump_hierarchy.call_count == 4
|
||||
|
||||
|
||||
def test_wait_timeout_aborts_gracefully(test_dumps):
|
||||
"""Test what happens if the network is so slow it times out entirely."""
|
||||
device = MagicMock()
|
||||
# Always return grid
|
||||
device.dump_hierarchy.return_value = test_dumps["grid"]
|
||||
|
||||
# Patch time.time to simulate 6 seconds passing immediately
|
||||
# We add sequence padding because python's logger internally uses time.time()
|
||||
with patch("time.time", side_effect=[0, 1, 6, 6, 6, 6, 6, 6, 6, 6]):
|
||||
with patch("GramAddict.core.bot_flow.sleep", return_value=None):
|
||||
success = _wait_for_post_loaded(device, timeout=5)
|
||||
assert success is False
|
||||
|
||||
|
||||
def test_empty_content_extraction_guard(test_dumps):
|
||||
"""
|
||||
Test that if a post is loaded, but it has strange empty text (foreign language or bug),
|
||||
the bot aborts interaction and scrolls instead of judging empty content.
|
||||
"""
|
||||
device = MagicMock()
|
||||
nav_graph = MagicMock()
|
||||
configs = ConfigMock()
|
||||
|
||||
# We create a fake active inference engine to just break the loop after 1 iteration
|
||||
ai = MagicMock()
|
||||
# Dopamine engine controls loop exit
|
||||
dopamine = MagicMock()
|
||||
dopamine.is_app_session_over.side_effect = [False, True] # Run once, then exit
|
||||
dopamine.wants_to_change_feed.return_value = False
|
||||
dopamine.wants_to_doomscroll.return_value = False
|
||||
|
||||
cognitive_stack = {
|
||||
"dopamine": dopamine,
|
||||
"active_inference": ai,
|
||||
"resonance": None,
|
||||
"growth_brain": None,
|
||||
"swarm": None,
|
||||
"darwin": None,
|
||||
}
|
||||
|
||||
# Mutate the post so it has NO text or description
|
||||
broken_xml = mutate_xml_to_foreign(test_dumps["post"])
|
||||
device.dump_hierarchy.return_value = broken_xml
|
||||
|
||||
from GramAddict.core.situational_awareness import SituationType
|
||||
|
||||
with (
|
||||
patch("GramAddict.core.bot_flow._humanized_scroll") as mock_scroll,
|
||||
patch("GramAddict.core.bot_flow.sleep"),
|
||||
patch(
|
||||
"GramAddict.core.situational_awareness.SituationalAwarenessEngine.perceive",
|
||||
return_value=SituationType.NORMAL,
|
||||
),
|
||||
):
|
||||
result = _run_zero_latency_feed_loop(device, None, nav_graph, configs, MagicMock(), "HomeFeed", cognitive_stack)
|
||||
|
||||
# Ensure scroll was called (the recovery mechanism)
|
||||
assert mock_scroll.called
|
||||
# Check that we never called resonance evaluation because we broke early
|
||||
assert not ai.predict_state.called
|
||||
assert result == "FEED_EXHAUSTED"
|
||||
|
||||
|
||||
def test_missing_feed_markers_guard(test_dumps):
|
||||
"""
|
||||
Test that if the UI is completely foreign (e.g., a system popup),
|
||||
the bot detects missing feed markers and scrolls to recover.
|
||||
"""
|
||||
device = MagicMock()
|
||||
configs = ConfigMock()
|
||||
|
||||
dopamine = MagicMock()
|
||||
dopamine.is_app_session_over.side_effect = [False, True]
|
||||
dopamine.wants_to_change_feed.return_value = False
|
||||
dopamine.wants_to_doomscroll.return_value = False
|
||||
|
||||
cognitive_stack = {"dopamine": dopamine, "growth_brain": None, "active_inference": None}
|
||||
|
||||
# Mutate XML to remove all FEED MARKERS
|
||||
alien_xml = mutate_xml_remove_feed_markers(test_dumps["post"])
|
||||
device.dump_hierarchy.return_value = alien_xml
|
||||
|
||||
with patch("GramAddict.core.bot_flow._humanized_scroll"), patch("GramAddict.core.bot_flow.sleep"):
|
||||
_run_zero_latency_feed_loop(device, None, MagicMock(), configs, MagicMock(), "HomeFeed", cognitive_stack)
|
||||
|
||||
|
||||
@patch("GramAddict.core.device_facade.u2")
|
||||
def test_xpath_watcher_initialization(mock_u2):
|
||||
"""
|
||||
Test fixing the critical watcher API bug.
|
||||
Ensures that device facade uses .watcher("name").when(xpath=...)
|
||||
"""
|
||||
mock_d = MagicMock()
|
||||
mock_u2.connect.return_value = mock_d
|
||||
|
||||
# Setup mock chain: deviceV2.watcher("crash_dialog").when(...)
|
||||
mock_watcher = MagicMock()
|
||||
mock_d.watcher.return_value = mock_watcher
|
||||
mock_when = MagicMock()
|
||||
mock_watcher.when.return_value = mock_when
|
||||
|
||||
# Just init the facade
|
||||
from GramAddict.core.device_facade import create_device
|
||||
|
||||
create_device("fake_serial", "com.fake.app", MagicMock())
|
||||
|
||||
# Verify exact API call structure for XPath
|
||||
mock_d.watcher.assert_any_call("crash_dialog")
|
||||
mock_d.watcher.assert_any_call("system_dialog")
|
||||
|
||||
# We can't perfectly assert the chained arguments natively without a bit of inspection,
|
||||
# but we can verify it didn't crash and called start
|
||||
assert mock_d.watcher.start.called
|
||||
@@ -1,50 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.device_facade import DeviceFacade
|
||||
|
||||
|
||||
def test_adb_retry_recovers_from_transient_error():
|
||||
# Attempt simulated disconnect on dump_hierarchy
|
||||
device_id = "test"
|
||||
app_id = "test"
|
||||
|
||||
with patch("uiautomator2.connect") as mock_connect:
|
||||
mock_device = MagicMock()
|
||||
mock_connect.return_value = mock_device
|
||||
|
||||
facade = DeviceFacade(device_id, app_id, None)
|
||||
|
||||
# Make the first 2 calls fail, the 3rd one pass
|
||||
mock_device.dump_hierarchy.side_effect = [
|
||||
Exception("ConnectError uiautomator2"),
|
||||
Exception("RPC Error"),
|
||||
"<hierarchy></hierarchy>",
|
||||
]
|
||||
|
||||
# Patch sleep to speed up test
|
||||
with patch("GramAddict.core.device_facade.sleep"):
|
||||
res = facade.dump_hierarchy()
|
||||
assert res == "<hierarchy></hierarchy>"
|
||||
assert mock_device.dump_hierarchy.call_count == 3
|
||||
|
||||
|
||||
def test_adb_retry_crashes_gracefully_after_all_retries():
|
||||
# Attempt simulated disconnect on dump_hierarchy
|
||||
device_id = "test"
|
||||
app_id = "test"
|
||||
|
||||
with patch("uiautomator2.connect") as mock_connect:
|
||||
mock_device = MagicMock()
|
||||
mock_connect.return_value = mock_device
|
||||
|
||||
facade = DeviceFacade(device_id, app_id, None)
|
||||
|
||||
# Always fail
|
||||
mock_device.dump_hierarchy.side_effect = Exception("Permanent ConnectError")
|
||||
|
||||
with patch("GramAddict.core.device_facade.sleep"):
|
||||
with pytest.raises(Exception, match="Permanent ConnectError"):
|
||||
facade.dump_hierarchy()
|
||||
assert mock_device.dump_hierarchy.call_count == 3
|
||||
@@ -1,96 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
# Add parent dir to path
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
class DummyDevice:
|
||||
class DeviceV2:
|
||||
def __init__(self):
|
||||
self.last_click = None
|
||||
|
||||
def click(self, x, y):
|
||||
self.last_click = (x, y)
|
||||
|
||||
def screenshot(self, path=None):
|
||||
return "fake_screenshot"
|
||||
|
||||
def __init__(self):
|
||||
import unittest
|
||||
|
||||
self.deviceV2 = self.DeviceV2()
|
||||
self.app_id = "com.instagram.android"
|
||||
self.args = unittest.mock.MagicMock()
|
||||
self.args.ai_telepathic_model = "qwen2.5:3b"
|
||||
self.args.ai_telepathic_url = "http://localhost:11434/api/generate"
|
||||
|
||||
def _get_current_app(self):
|
||||
return "com.instagram.android"
|
||||
|
||||
def get_info(self):
|
||||
return {"displayHeight": 2400, "displayWidth": 1080}
|
||||
|
||||
def screenshot(self, path=None):
|
||||
return "fake_screenshot"
|
||||
|
||||
|
||||
class TestHumanHesitation(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.telepathic = TelepathicEngine()
|
||||
self.device = DummyDevice()
|
||||
|
||||
def test_discard_dialog_extraction(self):
|
||||
"""
|
||||
Prove that the Telepathic Engine can correctly identify the 'Discard'
|
||||
button inside a synthetic XML dump, ensuring the 'Umentscheidung'
|
||||
abort logic works in the wild.
|
||||
"""
|
||||
# Synthetic Discard Dialog XML
|
||||
synthetic_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="0" bounds="[0,0][1080,2400]" package="com.instagram.android">
|
||||
<node index="1" class="android.widget.TextView" text="Discard Comment?" bounds="[200,1000][800,1100]" />
|
||||
<node index="2" class="android.widget.Button" text="IGNORE" content-desc="IGNORE" bounds="[200,1200][400,1300]" />
|
||||
<node index="3" class="android.widget.Button" text="Verwerfen" content-desc="Discard or Verwerfen popup button" bounds="[600,1200][800,1300]" resource-id="com.instagram.android:id/button_discard" />
|
||||
</node>
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
# Act
|
||||
result = self.telepathic.find_best_node(
|
||||
synthetic_dump,
|
||||
"Discard or Verwerfen popup button to cancel comment",
|
||||
device=self.device,
|
||||
min_confidence=0.5,
|
||||
)
|
||||
|
||||
# Assert (Should hit the [600,1200][800,1300] box, which centers to (700, 1250))
|
||||
self.assertIsNotNone(result, "Telepathic engine failed to find 'Verwerfen'.")
|
||||
self.assertEqual(result["x"], 700)
|
||||
self.assertEqual(result["y"], 1250)
|
||||
|
||||
def test_dm_inbox_tab_resolution(self):
|
||||
"""
|
||||
Verify that teleporting specifically to the Inbox tab (DM button)
|
||||
succeeds if 'Message' describes it.
|
||||
"""
|
||||
synthetic_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="2" text="" id="direct_tab" package="com.instagram.android" content-desc="Direct messages tab button" bounds="[432,2235][648,2361]" resource-id="com.instagram.android:id/direct_tab">
|
||||
<node content-desc=""/>
|
||||
</node>
|
||||
</hierarchy>"""
|
||||
|
||||
# If ID didn't match perfectly, we fall back to description as programmed.
|
||||
# Direct simulation of UI Automator check isn't in scope for this telepathic test,
|
||||
# but we can ensure Telepathic Engine CAN find it if we rely on it.
|
||||
result = self.telepathic.find_best_node(synthetic_dump, "Direct messages tab button", device=self.device)
|
||||
self.assertIsNotNone(result, "Should find the Message tab")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,52 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
|
||||
def test_query_llm_hallucination_recovery():
|
||||
# Test that when the primary model hallucinates non-JSON, it triggers fallback
|
||||
with patch("requests.post") as mock_post:
|
||||
# 1st call: Primary fails entirely (e.g., Timeout or strange error)
|
||||
mock_response_1 = MagicMock()
|
||||
mock_response_1.status_code = 500
|
||||
mock_response_1.raise_for_status.side_effect = Exception("500 Server Error")
|
||||
|
||||
# 2nd call: Fallback works and returns valid JSON
|
||||
mock_response_2 = MagicMock()
|
||||
mock_response_2.status_code = 200
|
||||
mock_response_2.raise_for_status.return_value = None
|
||||
mock_response_2.json.return_value = {"choices": [{"message": {"content": '{"test": "success"}'}}]}
|
||||
|
||||
mock_post.side_effect = [mock_response_1, mock_response_2]
|
||||
|
||||
# Attempt a query with a primary model
|
||||
res = query_llm(
|
||||
url="http://fake.api/v1/chat/completions",
|
||||
model="primary-model",
|
||||
prompt="Hello",
|
||||
format_json=True,
|
||||
fallback_model="fallback-model",
|
||||
fallback_url="http://fake.api/v1/chat/completions",
|
||||
)
|
||||
|
||||
assert res is not None
|
||||
assert "response" in res
|
||||
assert res["response"] == '{"test": "success"}'
|
||||
assert mock_post.call_count == 2
|
||||
|
||||
|
||||
def test_query_llm_double_hallucination_safe_return():
|
||||
# Test that when both models hallucinate, we return None gracefully
|
||||
with patch("requests.post") as mock_post:
|
||||
# Both models fail
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.raise_for_status.side_effect = Exception("500 Server Error")
|
||||
|
||||
mock_post.side_effect = [mock_response, mock_response]
|
||||
|
||||
res = query_llm(
|
||||
url="http://fake.api/v1/chat/completions", model="primary-model", prompt="Hello", format_json=True
|
||||
)
|
||||
|
||||
assert res is None
|
||||
@@ -1,49 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
|
||||
def test_tap_home_tab_recovery_from_homescreen():
|
||||
"""
|
||||
TDD: Reproduce the failure where tap_home_tab fails because the bot is on
|
||||
the Android Homescreen (app.lawnchair), and verify that it recovers
|
||||
via app_start instead of enterring an auto-repair loop.
|
||||
"""
|
||||
# 1. Setup Mock Device
|
||||
mock_device = MagicMock()
|
||||
mock_device.app_id = "com.instagram.android"
|
||||
# Return homescreen package to simulate context loss
|
||||
mock_device._get_current_app.return_value = "app.lawnchair"
|
||||
|
||||
# 2. Mock DeviceV2 responses
|
||||
mock_device.dump_hierarchy.return_value = "<hierarchy />"
|
||||
mock_device.app_start.return_value = True
|
||||
|
||||
# 3. Initialize NavGraph
|
||||
graph = QNavGraph(mock_device)
|
||||
graph.current_state = "ProfileFeed" # Assume stale state
|
||||
|
||||
# 4. Patch TelepathicEngine.get_instance to return a mock engine
|
||||
with (
|
||||
patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance") as mock_get_instance,
|
||||
patch("GramAddict.core.goap.PathMemory.learn_path"),
|
||||
patch("GramAddict.core.goap.PathMemory.recall_path", return_value=None),
|
||||
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB._get_embedding", return_value=[0] * 1536),
|
||||
patch(
|
||||
"GramAddict.core.situational_awareness.SituationalAwarenessEngine.ensure_clear_screen", return_value=False
|
||||
),
|
||||
patch("GramAddict.core.q_nav_graph.time.sleep"),
|
||||
):
|
||||
mock_engine = MagicMock()
|
||||
mock_get_instance.return_value = mock_engine
|
||||
|
||||
# Simulate Context Guard hitting: return None forever
|
||||
mock_engine.find_best_node.return_value = None
|
||||
|
||||
# 5. Execute
|
||||
# We expect this to return False gracefully after 3 attempts, without infinitely looping
|
||||
success = graph.navigate_to("ExploreFeed", zero_engine=None)
|
||||
|
||||
# 6. Assertion
|
||||
assert not success, "Navigation should fail gracefully when context cannot be recovered"
|
||||
assert mock_device.app_start.called, "Should have force-started the app when context was lost"
|
||||
@@ -1,123 +0,0 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Force mock qdrant_client before importing any core modules that depend on it
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
|
||||
|
||||
class TestQNavGraphEdgeCases:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_graph(self):
|
||||
self.device = MagicMock()
|
||||
self.device.app_id = "com.instagram.android"
|
||||
self.device.info = {"screenOn": True}
|
||||
self.device.dump_hierarchy.return_value = '<hierarchy><node package="com.instagram.android" /></hierarchy>'
|
||||
self.device._get_current_app = MagicMock(return_value="com.instagram.android")
|
||||
|
||||
# Prevent Dojo engine instantiation during tests
|
||||
with patch("GramAddict.core.compiler_engine.VLMCompilerEngine"):
|
||||
self.graph = QNavGraph(self.device)
|
||||
|
||||
def test_find_path_edge_cases(self):
|
||||
# 1. Start == End
|
||||
assert self.graph._find_path("HomeFeed", "HomeFeed") == []
|
||||
|
||||
# 2. Start not in nodes
|
||||
assert self.graph._find_path("UnknownState", "HomeFeed") is None
|
||||
|
||||
# 3. Unreachable states
|
||||
self.graph.nodes = {
|
||||
"HomeFeed": {"transitions": {"tap_explore": "ExploreFeed"}},
|
||||
"IsolatedFeed": {"transitions": {}},
|
||||
}
|
||||
assert self.graph._find_path("HomeFeed", "IsolatedFeed") is None
|
||||
|
||||
# 4. Infinite loop protection (A -> B -> A)
|
||||
self.graph.nodes = {"A": {"transitions": {"to_b": "B"}}, "B": {"transitions": {"to_a": "A"}}}
|
||||
assert (
|
||||
self.graph._find_path("A", "C") is None
|
||||
) # Should safely return None without exceeding recursion/loop depth
|
||||
|
||||
# 5. Longest path possible before unreachability is confirmed
|
||||
assert self.graph._find_path("B", "D") is None
|
||||
|
||||
# 6. Diamond shape path
|
||||
self.graph.nodes = {
|
||||
"Start": {"transitions": {"top": "Top", "bottom": "Bottom"}},
|
||||
"Top": {"transitions": {"top_to_end": "End"}},
|
||||
"Bottom": {"transitions": {"bottom_to_end": "End"}},
|
||||
"End": {},
|
||||
}
|
||||
# BFS should find shortest path (len 2)
|
||||
assert len(self.graph._find_path("Start", "End")) == 2
|
||||
|
||||
@patch("GramAddict.core.q_nav_graph.time.sleep", return_value=None)
|
||||
@patch("GramAddict.core.q_nav_graph.random_sleep", return_value=None)
|
||||
@patch("GramAddict.core.situational_awareness.random_sleep", return_value=None)
|
||||
@patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance")
|
||||
def test_execute_transition_edge_cases(self, mock_get_telepathic, mock_sae_sleep, mock_q_rand_sleep, mock_q_sleep):
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
mock_engine = MagicMock(spec=TelepathicEngine)
|
||||
mock_get_telepathic.return_value = mock_engine
|
||||
|
||||
# Case 1: Telepathic engine finds nothing
|
||||
mock_engine.find_best_node.return_value = None
|
||||
|
||||
# If still in Instagram, it returns False
|
||||
self.device._get_current_app.return_value = "com.instagram.android"
|
||||
assert not self.graph._execute_transition("unknown_action", mock_engine)
|
||||
|
||||
# If app is different, it returns "CONTEXT_LOST"
|
||||
self.device._get_current_app.return_value = "com.android.launcher3"
|
||||
assert self.graph._execute_transition("unknown_action", mock_engine) == "CONTEXT_LOST"
|
||||
|
||||
# Case 2: Best node has skip flag
|
||||
mock_engine.find_best_node.return_value = {"skip": True}
|
||||
assert self.graph._execute_transition("already_done_action", mock_engine)
|
||||
|
||||
# Case 3: Proper interaction, but XML doesn't change (verification fail)
|
||||
mock_engine.find_best_node.return_value = {"x": 10, "y": 10, "score": 0.9}
|
||||
same_xml = '<hierarchy><node package="com.instagram.android" class="same" /></hierarchy>'
|
||||
self.device.dump_hierarchy.side_effect = None
|
||||
self.device.dump_hierarchy.return_value = same_xml
|
||||
assert not self.graph._execute_transition("click_action", mock_engine)
|
||||
assert mock_engine.reject_click.call_count == 3
|
||||
|
||||
# Case 4: Proper interaction, XML changes (verification pass)
|
||||
mock_engine.reset_mock()
|
||||
mock_engine.find_best_node.return_value = {"x": 10, "y": 10, "score": 0.9}
|
||||
before_xml = '<hierarchy><node package="com.instagram.android" class="before" /></hierarchy>'
|
||||
after_xml = '<hierarchy><node package="com.instagram.android" class="after" /></hierarchy>'
|
||||
|
||||
initial_clicks = self.device.click.call_count
|
||||
|
||||
def dynamic_xml(*args, **kwargs):
|
||||
return after_xml if self.device.click.call_count > initial_clicks else before_xml
|
||||
|
||||
self.device.dump_hierarchy.side_effect = dynamic_xml
|
||||
# Explicitly ensure verify_success is truthy
|
||||
mock_engine.verify_success.return_value = True
|
||||
|
||||
assert self.graph._execute_transition("click_action", mock_engine)
|
||||
mock_engine.confirm_click.assert_called_once()
|
||||
|
||||
@patch("GramAddict.core.q_nav_graph.time.sleep", return_value=None)
|
||||
@patch("GramAddict.core.q_nav_graph.random_sleep", return_value=None)
|
||||
@patch("GramAddict.core.situational_awareness.random_sleep", return_value=None)
|
||||
@patch("GramAddict.core.dojo_engine.DojoEngine.get_instance")
|
||||
def test_navigate_to_recovery_edge_cases(self, mock_dojo, mock_sae_sleep, mock_q_rand_sleep, mock_q_sleep):
|
||||
# We test the deepest recovery logic: when everything fails
|
||||
|
||||
zero_engine = MagicMock()
|
||||
|
||||
# Mock transitions completely failing
|
||||
with patch.object(self.graph.goap, "navigate_to_screen", return_value=False):
|
||||
# Recovery attempts maxed out
|
||||
assert not self.graph.navigate_to("ExploreFeed", zero_engine, recovery_attempts=3)
|
||||
|
||||
# Start logic where path is None and direct fallback also fails
|
||||
self.graph.current_state = "IsolatedNode"
|
||||
# It should trigger fallback and then return False because `navigate_to_screen` always returns False
|
||||
assert not self.graph.navigate_to("ExploreFeed", zero_engine, recovery_attempts=0)
|
||||
@@ -1,59 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")))
|
||||
|
||||
from GramAddict.core.goap import GoalExecutor, ScreenType
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_device():
|
||||
device = MagicMock()
|
||||
# Simulate XML changing but screen type not being the target
|
||||
device.dump_hierarchy.side_effect = ["<xml1/>", "<xml2/>", "<xml2/>"]
|
||||
device.app_id = "com.instagram.android"
|
||||
return device
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_telepathic():
|
||||
with patch("GramAddict.core.telepathic_engine.TelepathicEngine.get_instance") as mock:
|
||||
engine = mock.return_value
|
||||
engine.find_best_node.return_value = {"x": 100, "y": 200, "semantic_string": "mock_node"}
|
||||
yield engine
|
||||
|
||||
|
||||
def test_execution_rejects_wrong_screen(mock_device, mock_telepathic):
|
||||
"""
|
||||
TDD Case: If we intend to go to DMs but land on Reels,
|
||||
TelepathicEngine.confirm_click should NOT be called.
|
||||
"""
|
||||
executor = GoalExecutor(mock_device, "testuser")
|
||||
|
||||
# We mock perceive to return ReelsFeed after the click
|
||||
with patch.object(executor, "perceive") as mock_perceive:
|
||||
# Before click
|
||||
mock_perceive.side_effect = [
|
||||
{"screen_type": ScreenType.HOME_FEED}, # Initial
|
||||
{"screen_type": ScreenType.REELS_FEED}, # After click (WRONG!)
|
||||
]
|
||||
|
||||
# Action that intends to go to DM_INBOX
|
||||
action = "tap messages tab"
|
||||
|
||||
# We need to make sure _execute_action knows the goal is "open messages"
|
||||
# Since _execute_action is usually called from achieve(), we mock that flow
|
||||
|
||||
success = executor._execute_action(action, goal="open messages")
|
||||
|
||||
# Success should be False because we didn't reach the goal
|
||||
# (Or True if we only care about XML change, but that's what we're changing)
|
||||
assert success is False
|
||||
|
||||
# CRITICAL: confirm_click should NOT have been called for 'messages tab'
|
||||
# since we are on Reels.
|
||||
mock_telepathic.confirm_click.assert_not_called()
|
||||
mock_telepathic.reject_click.assert_called_once_with(action)
|
||||
@@ -1,68 +0,0 @@
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
class TestTelepathicGuards:
|
||||
def setup_method(self):
|
||||
self.engine = TelepathicEngine()
|
||||
|
||||
def test_strict_story_ring_guard(self):
|
||||
"""
|
||||
TDD: Story rings MUST be physically near the top of the screen (y < 30%).
|
||||
Post profile headers that appear further down must be aggressively blocked
|
||||
when the intent is 'tap story ring avatar'.
|
||||
"""
|
||||
intent = "tap story ring avatar"
|
||||
screen_height = 2400
|
||||
|
||||
# Valid Story Ring (Top of screen, but below status bar)
|
||||
valid_story = {"resource_id": "reel_ring", "y": 300, "area": 100}
|
||||
assert self.engine._structural_sanity_check(valid_story, intent, screen_height) is True
|
||||
|
||||
# Invalid Story Ring (Hallucination: Post profile header in the feed)
|
||||
invalid_story = {"resource_id": "row_feed_profile_header", "y": 800, "area": 100}
|
||||
assert self.engine._structural_sanity_check(invalid_story, intent, screen_height) is False
|
||||
|
||||
def test_strict_button_guard(self):
|
||||
"""
|
||||
TDD: When explicitly looking for a 'button', nodes that declare themselves
|
||||
as profiles (e.g. 'go to profile') must be blocked, to prevent accidental
|
||||
profile visits when clicking 'like'.
|
||||
"""
|
||||
intent = "Heart like button for comment"
|
||||
screen_height = 2400
|
||||
|
||||
# Valid Like Button
|
||||
valid_btn = {"resource_id": "like_button", "semantic_string": "Like", "y": 1000, "area": 100}
|
||||
assert self.engine._structural_sanity_check(valid_btn, intent, screen_height) is True
|
||||
|
||||
# Invalid Profile Link masquerading as a match due to string proximity
|
||||
invalid_prof = {
|
||||
"resource_id": "username",
|
||||
"semantic_string": "Go to cayleighanddavid's profile",
|
||||
"y": 1000,
|
||||
"area": 100,
|
||||
}
|
||||
assert self.engine._structural_sanity_check(invalid_prof, intent, screen_height) is False
|
||||
|
||||
# However, if the intent *is* profile, it should pass
|
||||
intent_prof = "go to profile"
|
||||
assert self.engine._structural_sanity_check(invalid_prof, intent_prof, screen_height) is True
|
||||
|
||||
def test_like_semantic_verification(self):
|
||||
"""
|
||||
TDD: Verify that 'unlike' is treated as a successful 'Like' action,
|
||||
because tapping 'Like' changes the state to 'Unlike' in English Instagram.
|
||||
"""
|
||||
# Testing the specific regex logic inside verify_success
|
||||
import re
|
||||
|
||||
xml_dump_success = '<node class="android.widget.ImageView" content-desc="Unlike" />'
|
||||
|
||||
marker_found = re.search(r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_success.lower())
|
||||
assert marker_found is not None
|
||||
|
||||
xml_dump_fail = '<node class="android.widget.ImageView" content-desc="Like" />'
|
||||
marker_found_fail = re.search(
|
||||
r"\b(liked|unlike|gefällt mir nicht mehr|gefällt mir am)\b", xml_dump_fail.lower()
|
||||
)
|
||||
assert marker_found_fail is None
|
||||
@@ -1,67 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")))
|
||||
|
||||
from GramAddict.core.q_nav_graph import QNavGraph
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
|
||||
class TestTrapEscape(unittest.TestCase):
|
||||
@patch("GramAddict.core.q_nav_graph.time.sleep", return_value=None)
|
||||
@patch("GramAddict.core.q_nav_graph.random_sleep", return_value=None)
|
||||
@patch("GramAddict.core.situational_awareness.SituationalAwarenessEngine.ensure_clear_screen", return_value=False)
|
||||
def test_trap_guard_autonomous_ai_escape(self, mock_sae_clear, mock_q_rand_sleep, mock_q_sleep):
|
||||
print("Starting TDD: Testing autonomous Trap Escape with semantic bypass...")
|
||||
|
||||
# 1. Setup mocks
|
||||
mock_device = MagicMock()
|
||||
mock_device.app_id = "com.instagram.android"
|
||||
mock_device._get_current_app.return_value = "com.instagram.android"
|
||||
|
||||
trap_xml = "<hierarchy><node resource-id='modal_trap' /></hierarchy>"
|
||||
current_xml = [trap_xml]
|
||||
|
||||
# Dynamic dump that changes after click
|
||||
def dynamic_dump():
|
||||
return current_xml[0]
|
||||
|
||||
def dynamic_click(**kwargs):
|
||||
if kwargs.get("obj") and kwargs["obj"].get("semantic") and "done" in kwargs["obj"].get("semantic").lower():
|
||||
current_xml[0] = "<html><node text='Reels'/><node text='Home'/></html>"
|
||||
|
||||
mock_device.dump_hierarchy.side_effect = dynamic_dump
|
||||
mock_device.click.side_effect = dynamic_click
|
||||
|
||||
nav_graph = QNavGraph(device=mock_device)
|
||||
|
||||
engine = TelepathicEngine.get_instance()
|
||||
engine.confirm_click = MagicMock()
|
||||
engine.reject_click = MagicMock()
|
||||
|
||||
original_find_best_node = engine.find_best_node
|
||||
|
||||
def spy_find_best_node(xml_hierarchy, intent_description, **kwargs):
|
||||
if "tap home tab" in intent_description.lower():
|
||||
return None
|
||||
return original_find_best_node(xml_hierarchy, intent_description, **kwargs)
|
||||
|
||||
engine.find_best_node = spy_find_best_node
|
||||
nav_graph.engine = engine # explicitly enforce
|
||||
|
||||
# 2. Execute transition
|
||||
# Mock engine finds nothing, triggering the final fallback escape
|
||||
nav_graph._execute_transition("tap_home_tab", max_retries=1, mock_semantic_engine=engine)
|
||||
|
||||
# 3. Assertions
|
||||
# The new SAE/nav_graph behavior explicitly presses BACK when 'tap_home_tab' fails after all retries
|
||||
self.assertTrue(mock_device.press.called, "Trap guard did not autonomously press BACK to escape the sub-view!")
|
||||
called_key = mock_device.press.call_args_list[0][0][0]
|
||||
self.assertEqual(called_key, "back")
|
||||
print("TDD SUCCESS: Autonomous Backend fallback confirmed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
BIN
tests/chaos/__pycache__/__init__.cpython-311.pyc
Normal file
BIN
tests/chaos/__pycache__/__init__.cpython-311.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,227 +0,0 @@
|
||||
"""
|
||||
Chaos Engineering: Network & Dependency Failure Tests.
|
||||
|
||||
Verifies that the bot degrades gracefully when external services
|
||||
(Qdrant, Ollama, OpenRouter) are unavailable, slow, or return errors.
|
||||
|
||||
Tesla's FSD doesn't crash if the map server is unreachable — neither should we.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.chaos import VALID_FEED_XML
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# Qdrant Failure Tests
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.chaos
|
||||
class TestQdrantFailure:
|
||||
"""Bot must survive total Qdrant outage."""
|
||||
|
||||
def test_telepathic_works_without_qdrant(self):
|
||||
"""TelepathicEngine must still resolve nodes via keyword fast-path when Qdrant is down."""
|
||||
with (
|
||||
patch("GramAddict.core.qdrant_memory.QdrantBase.__init__", return_value=None),
|
||||
patch(
|
||||
"GramAddict.core.qdrant_memory.QdrantBase.is_connected",
|
||||
new_callable=lambda: property(lambda self: False),
|
||||
),
|
||||
):
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
TelepathicEngine._instance = None
|
||||
engine = TelepathicEngine.__new__(TelepathicEngine)
|
||||
engine.ui_memory = MagicMock()
|
||||
engine.ui_memory.is_connected = False
|
||||
engine.ui_memory.query_closest = MagicMock(return_value=None)
|
||||
engine.positive_memory = MagicMock()
|
||||
engine.positive_memory.is_connected = False
|
||||
engine.positive_memory.recall = MagicMock(return_value=None)
|
||||
engine._edge_model = None
|
||||
engine._edge_tokenizer = None
|
||||
|
||||
nodes = engine._extract_semantic_nodes(VALID_FEED_XML)
|
||||
# Should still find clickable nodes via structural parsing
|
||||
assert len(nodes) > 0
|
||||
TelepathicEngine._instance = None
|
||||
|
||||
def test_sae_recall_returns_none_without_qdrant(self):
|
||||
"""SAE episodic memory must return None (not crash) when Qdrant is down."""
|
||||
with (
|
||||
patch("GramAddict.core.qdrant_memory.QdrantBase.__init__", return_value=None),
|
||||
patch(
|
||||
"GramAddict.core.qdrant_memory.QdrantBase.is_connected",
|
||||
new_callable=lambda: property(lambda self: False),
|
||||
),
|
||||
):
|
||||
from GramAddict.core.situational_awareness import SituationEpisodeDB
|
||||
|
||||
db = SituationEpisodeDB()
|
||||
db._db = MagicMock()
|
||||
db._db.is_connected = False
|
||||
|
||||
result = db.recall("test_situation_signature")
|
||||
assert result is None
|
||||
|
||||
def test_sae_learn_silently_fails_without_qdrant(self):
|
||||
"""SAE learning must silently skip (not crash) when Qdrant is down."""
|
||||
with (
|
||||
patch("GramAddict.core.qdrant_memory.QdrantBase.__init__", return_value=None),
|
||||
patch(
|
||||
"GramAddict.core.qdrant_memory.QdrantBase.is_connected",
|
||||
new_callable=lambda: property(lambda self: False),
|
||||
),
|
||||
):
|
||||
from GramAddict.core.situational_awareness import EscapeAction, SituationEpisodeDB
|
||||
|
||||
db = SituationEpisodeDB()
|
||||
db._db = MagicMock()
|
||||
db._db.is_connected = False
|
||||
|
||||
action = EscapeAction("back", reason="test")
|
||||
# Must not raise
|
||||
db.learn("test_signature", action, True)
|
||||
|
||||
def test_qdrant_timeout_doesnt_hang_extraction(self):
|
||||
"""If Qdrant queries time out, node extraction must still complete."""
|
||||
import time
|
||||
|
||||
with (
|
||||
patch("GramAddict.core.qdrant_memory.QdrantBase.__init__", return_value=None),
|
||||
patch(
|
||||
"GramAddict.core.qdrant_memory.QdrantBase.is_connected",
|
||||
new_callable=lambda: property(lambda self: False),
|
||||
),
|
||||
):
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
TelepathicEngine._instance = None
|
||||
engine = TelepathicEngine.__new__(TelepathicEngine)
|
||||
engine.ui_memory = MagicMock()
|
||||
engine.ui_memory.is_connected = False
|
||||
engine.ui_memory.query_closest = MagicMock(side_effect=TimeoutError("Qdrant timeout"))
|
||||
engine.positive_memory = MagicMock()
|
||||
engine.positive_memory.is_connected = False
|
||||
engine.positive_memory.recall = MagicMock(side_effect=TimeoutError("Qdrant timeout"))
|
||||
engine._edge_model = None
|
||||
engine._edge_tokenizer = None
|
||||
|
||||
start = time.time()
|
||||
nodes = engine._extract_semantic_nodes(VALID_FEED_XML)
|
||||
elapsed = time.time() - start
|
||||
|
||||
assert elapsed < 5.0
|
||||
assert isinstance(nodes, list)
|
||||
TelepathicEngine._instance = None
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# LLM (Ollama/OpenRouter) Failure Tests
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.chaos
|
||||
class TestLLMFailure:
|
||||
"""Bot must survive LLM outages."""
|
||||
|
||||
def test_sae_perceive_defaults_to_normal_on_llm_failure(self):
|
||||
"""If LLM classification fails, SAE must default to NORMAL (safe fallback)."""
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device.deviceV2 = MagicMock()
|
||||
device.deviceV2.info = {"screenOn": True}
|
||||
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
sae.episodes = MagicMock()
|
||||
sae.episodes.recall = MagicMock(return_value=None)
|
||||
|
||||
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB") as MockScreenDB:
|
||||
mock_screen_db = MagicMock()
|
||||
mock_screen_db.get_screen_type = MagicMock(return_value=None)
|
||||
MockScreenDB.return_value = mock_screen_db
|
||||
|
||||
with patch("GramAddict.core.llm_provider.query_telepathic_llm", side_effect=ConnectionError("Ollama down")):
|
||||
result = sae.perceive(VALID_FEED_XML)
|
||||
# Must default to NORMAL, not crash
|
||||
assert result == SituationType.NORMAL
|
||||
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
def test_sae_escape_planning_defaults_to_back_on_llm_failure(self):
|
||||
"""If LLM escape planning fails, SAE must default to BACK press."""
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
|
||||
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device.deviceV2 = MagicMock()
|
||||
device.deviceV2.info = {"screenOn": True}
|
||||
|
||||
sae = SituationalAwarenessEngine(device)
|
||||
|
||||
with patch("GramAddict.core.llm_provider.query_llm", side_effect=ConnectionError("LLM down")):
|
||||
action = sae._plan_escape_via_llm(VALID_FEED_XML, "compressed_sig", SituationType.OBSTACLE_MODAL)
|
||||
assert action.action_type == "back"
|
||||
assert "failed" in action.reason.lower() or "default" in action.reason.lower()
|
||||
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# Active Inference Resilience
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.chaos
|
||||
class TestActiveInferenceChaos:
|
||||
"""Active Inference engine must survive edge cases."""
|
||||
|
||||
def test_evaluate_with_empty_history(self):
|
||||
"""Evaluating without any predictions must return True (no-op)."""
|
||||
from GramAddict.core.active_inference import ActiveInferenceEngine
|
||||
|
||||
ai = ActiveInferenceEngine("test_user")
|
||||
assert ai.evaluate_prediction("<hierarchy/>") is True
|
||||
|
||||
def test_extreme_free_energy_doesnt_overflow(self):
|
||||
"""Repeated errors must not cause float overflow."""
|
||||
from GramAddict.core.active_inference import ActiveInferenceEngine
|
||||
|
||||
ai = ActiveInferenceEngine("test_user")
|
||||
|
||||
for _ in range(1000):
|
||||
ai.predict_state(["nonexistent_element"])
|
||||
ai.evaluate_prediction("<hierarchy><node text='wrong'/></hierarchy>")
|
||||
|
||||
assert ai.free_energy < float("inf")
|
||||
assert ai.free_energy >= 0
|
||||
|
||||
def test_surprise_with_identical_prediction_is_zero(self):
|
||||
"""Perfect prediction (predicted == observed) must produce near-zero surprise."""
|
||||
from GramAddict.core.active_inference import ActiveInferenceEngine
|
||||
|
||||
ai = ActiveInferenceEngine("test_user")
|
||||
ai.free_energy = 0.0
|
||||
|
||||
result = ai.calculate_surprise(1.0, 1.0)
|
||||
assert result < 0.1 # Near-zero free energy
|
||||
|
||||
def test_sleep_modifier_bounds(self):
|
||||
"""Sleep modifier must always be between 1.0 and 5.0."""
|
||||
from GramAddict.core.active_inference import ActiveInferenceEngine
|
||||
|
||||
ai = ActiveInferenceEngine("test_user")
|
||||
|
||||
for policy in ["STABLE", "CAUTIOUS", "DORMANT"]:
|
||||
ai.policy = policy
|
||||
mod = ai.get_sleep_modifier()
|
||||
assert 1.0 <= mod <= 5.0
|
||||
@@ -1,242 +0,0 @@
|
||||
"""
|
||||
Chaos Engineering: XML Corruption Resilience Tests for TelepathicEngine + SAE.
|
||||
|
||||
Verifies that NEITHER engine crashes on any form of corrupted, truncated,
|
||||
adversarial, or garbage XML input. They must degrade gracefully (return None
|
||||
or empty lists) without raising unhandled exceptions.
|
||||
|
||||
These tests are the "crash barrier" of autonomous navigation — ensuring that
|
||||
no matter what Android dumps to us, the bot survives and recovers.
|
||||
"""
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.chaos import generate_corrupted_xml
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# Telepathic Engine Chaos Tests
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def telepathic_engine():
|
||||
"""Creates a real TelepathicEngine instance with mocked Qdrant."""
|
||||
with (
|
||||
patch("GramAddict.core.qdrant_memory.QdrantBase.__init__", return_value=None),
|
||||
patch(
|
||||
"GramAddict.core.qdrant_memory.QdrantBase.is_connected", new_callable=lambda: property(lambda self: False)
|
||||
),
|
||||
):
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
TelepathicEngine._instance = None
|
||||
engine = TelepathicEngine.__new__(TelepathicEngine)
|
||||
engine.ui_memory = MagicMock()
|
||||
engine.ui_memory.is_connected = False
|
||||
engine.ui_memory.query_closest = MagicMock(return_value=None)
|
||||
engine.positive_memory = MagicMock()
|
||||
engine.positive_memory.is_connected = False
|
||||
engine.positive_memory.recall = MagicMock(return_value=None)
|
||||
engine._edge_model = None
|
||||
engine._edge_tokenizer = None
|
||||
yield engine
|
||||
TelepathicEngine._instance = None
|
||||
|
||||
|
||||
ALL_CORRUPTION_TYPES = [
|
||||
"EMPTY_STRING",
|
||||
"NONE_VALUE",
|
||||
"TRUNCATED_MID_TAG",
|
||||
"UNICODE_INJECTION",
|
||||
"MASSIVE_DOM_10K_NODES",
|
||||
"ZERO_SIZE_BOUNDS",
|
||||
"NEGATIVE_COORDINATES",
|
||||
"MISSING_CLOSING_TAGS",
|
||||
"RECURSIVE_NESTING_500_DEEP",
|
||||
"NULL_BYTES",
|
||||
"MALFORMED_BOUNDS",
|
||||
"ONLY_WHITESPACE",
|
||||
"HTML_NOT_XML",
|
||||
"BINARY_GARBAGE",
|
||||
"EXTREMELY_LONG_TEXT",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.chaos
|
||||
class TestTelepathicEngineChaos:
|
||||
"""Telepathic Engine must NEVER crash on corrupted XML."""
|
||||
|
||||
@pytest.mark.parametrize("corruption_type", ALL_CORRUPTION_TYPES)
|
||||
def test_extract_semantic_nodes_survives(self, telepathic_engine, corruption_type):
|
||||
"""Engine's XML parser must return empty list on any corruption."""
|
||||
xml = generate_corrupted_xml(corruption_type)
|
||||
|
||||
# Must NOT raise. May return empty list.
|
||||
if xml is None:
|
||||
# None input — directly test defense
|
||||
result = telepathic_engine._extract_semantic_nodes("")
|
||||
else:
|
||||
result = telepathic_engine._extract_semantic_nodes(xml)
|
||||
|
||||
assert isinstance(result, list)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corruption_type",
|
||||
[
|
||||
"EMPTY_STRING",
|
||||
"NONE_VALUE",
|
||||
"TRUNCATED_MID_TAG",
|
||||
"MISSING_CLOSING_TAGS",
|
||||
"ONLY_WHITESPACE",
|
||||
"HTML_NOT_XML",
|
||||
"BINARY_GARBAGE",
|
||||
],
|
||||
)
|
||||
def test_find_best_node_survives_garbage(self, telepathic_engine, corruption_type):
|
||||
"""find_best_node must return None on garbage XML, never crash."""
|
||||
xml = generate_corrupted_xml(corruption_type)
|
||||
if xml is None:
|
||||
xml = ""
|
||||
|
||||
result = telepathic_engine._find_best_node_inner(xml, "tap like button", min_confidence=0.82)
|
||||
# Must be None or a dict, never an exception
|
||||
assert result is None or isinstance(result, dict)
|
||||
|
||||
def test_unicode_injection_doesnt_corrupt_semantics(self, telepathic_engine):
|
||||
"""Zalgo text in nodes shouldn't crash semantic extraction."""
|
||||
xml = generate_corrupted_xml("UNICODE_INJECTION")
|
||||
nodes = telepathic_engine._extract_semantic_nodes(xml)
|
||||
# Should extract SOME nodes (the XML structure is valid)
|
||||
assert isinstance(nodes, list)
|
||||
# If nodes found, they should have valid coordinates
|
||||
for node in nodes:
|
||||
assert isinstance(node.get("x", 0), int)
|
||||
assert isinstance(node.get("y", 0), int)
|
||||
|
||||
def test_massive_dom_doesnt_hang(self, telepathic_engine):
|
||||
"""10K nodes must be parsed within 5 seconds — no infinite loops."""
|
||||
xml = generate_corrupted_xml("MASSIVE_DOM_10K_NODES")
|
||||
start = time.time()
|
||||
nodes = telepathic_engine._extract_semantic_nodes(xml)
|
||||
elapsed = time.time() - start
|
||||
|
||||
assert elapsed < 5.0, f"Parsing 10K nodes took {elapsed:.2f}s (limit: 5s)"
|
||||
assert isinstance(nodes, list)
|
||||
|
||||
def test_deep_nesting_doesnt_stackoverflow(self, telepathic_engine):
|
||||
"""500 levels of nesting must not cause stack overflow."""
|
||||
xml = generate_corrupted_xml("RECURSIVE_NESTING_500_DEEP")
|
||||
# This would crash Python's default recursion limit (1000) if
|
||||
# we used recursive parsing. ElementTree uses iterative parsing,
|
||||
# so it should survive.
|
||||
nodes = telepathic_engine._extract_semantic_nodes(xml)
|
||||
assert isinstance(nodes, list)
|
||||
|
||||
def test_null_bytes_stripped(self, telepathic_engine):
|
||||
"""Null bytes in text content must not cause parsing failures."""
|
||||
xml = generate_corrupted_xml("NULL_BYTES")
|
||||
nodes = telepathic_engine._extract_semantic_nodes(xml)
|
||||
assert isinstance(nodes, list)
|
||||
# Verify no null bytes leaked into node semantics
|
||||
for node in nodes:
|
||||
assert "\x00" not in node.get("semantic_string", "")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# SAE (Situational Awareness Engine) Chaos Tests
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sae_engine():
|
||||
"""Creates a SAE instance with mocked device."""
|
||||
from GramAddict.core.situational_awareness import SituationalAwarenessEngine
|
||||
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
device = MagicMock()
|
||||
device.app_id = "com.instagram.android"
|
||||
device.deviceV2 = MagicMock()
|
||||
device.deviceV2.info = {"screenOn": True}
|
||||
|
||||
engine = SituationalAwarenessEngine(device)
|
||||
|
||||
# Mock the episode DB to avoid Qdrant dependency
|
||||
engine.episodes = MagicMock()
|
||||
engine.episodes.recall = MagicMock(return_value=None)
|
||||
engine.episodes.learn = MagicMock()
|
||||
|
||||
yield engine
|
||||
SituationalAwarenessEngine.reset()
|
||||
|
||||
|
||||
@pytest.mark.chaos
|
||||
class TestSAEChaos:
|
||||
"""SAE perception must be bulletproof against XML corruption."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corruption_type",
|
||||
[
|
||||
"EMPTY_STRING",
|
||||
"TRUNCATED_MID_TAG",
|
||||
"MISSING_CLOSING_TAGS",
|
||||
"ONLY_WHITESPACE",
|
||||
"HTML_NOT_XML",
|
||||
"BINARY_GARBAGE",
|
||||
],
|
||||
)
|
||||
def test_compress_xml_survives_garbage(self, sae_engine, corruption_type):
|
||||
"""XML compression must never crash, even on garbage."""
|
||||
xml = generate_corrupted_xml(corruption_type)
|
||||
if xml is None:
|
||||
xml = ""
|
||||
|
||||
result = sae_engine._compress_xml(xml)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0 # Should always return something
|
||||
|
||||
def test_compress_empty_returns_marker(self, sae_engine):
|
||||
"""Empty/None input must return 'EMPTY_SCREEN' sentinel."""
|
||||
assert sae_engine._compress_xml("") == "EMPTY_SCREEN"
|
||||
assert sae_engine._compress_xml(None) == "EMPTY_SCREEN"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corruption_type",
|
||||
[
|
||||
"EMPTY_STRING",
|
||||
"TRUNCATED_MID_TAG",
|
||||
"BINARY_GARBAGE",
|
||||
"ONLY_WHITESPACE",
|
||||
],
|
||||
)
|
||||
def test_perceive_survives_garbage(self, sae_engine, corruption_type):
|
||||
"""perceive() must return a valid SituationType on any input."""
|
||||
from GramAddict.core.situational_awareness import SituationType
|
||||
|
||||
xml = generate_corrupted_xml(corruption_type)
|
||||
if xml is None:
|
||||
xml = ""
|
||||
|
||||
result = sae_engine.perceive(xml)
|
||||
assert isinstance(result, SituationType)
|
||||
|
||||
def test_compute_situation_hash_is_deterministic(self, sae_engine):
|
||||
"""Same XML must always produce the same hash."""
|
||||
xml = generate_corrupted_xml("UNICODE_INJECTION")
|
||||
compressed = sae_engine._compress_xml(xml)
|
||||
hash1 = sae_engine._compute_situation_hash(compressed)
|
||||
hash2 = sae_engine._compute_situation_hash(compressed)
|
||||
assert hash1 == hash2
|
||||
|
||||
def test_massive_dom_compression_is_bounded(self, sae_engine):
|
||||
"""10K nodes must be compressed to < 3000 chars (the cap)."""
|
||||
xml = generate_corrupted_xml("MASSIVE_DOM_10K_NODES")
|
||||
start = time.time()
|
||||
result = sae_engine._compress_xml(xml)
|
||||
elapsed = time.time() - start
|
||||
|
||||
assert len(result) <= 3000, f"Compressed output is {len(result)} chars (limit: 3000)"
|
||||
assert elapsed < 5.0, f"Compression took {elapsed:.2f}s"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user