fix(perception): complete bug 5,6,7,8,10 fixes
- Resolved Bug #5: Fixed list vs str parsing in ResonanceEvaluator - Resolved Bug #6: Use 'should_like' key for vibe score - Resolved Bug #7: Guard TelepathicEngine against 'Follow' nodes for post media - Resolved Bug #8: Implemented failed_bounds exclusion loop breaker in PerfectSnapping - Resolved Bug #10: Corrected available_actions string parsing - Validated with E2E regression suite (100% green)
This commit is contained in:
@@ -49,16 +49,26 @@ class ResonanceEvaluatorPlugin(BehaviorPlugin):
|
||||
tele = ctx.cognitive_stack.get("telepathic")
|
||||
if tele:
|
||||
logger.info("✨ [Resonance] Performing visual vibe check...")
|
||||
persona_interests = getattr(ctx.configs.args, "persona_interests", [])
|
||||
|
||||
# BUG 5 Fix: Read target_audience or persona_interests
|
||||
raw_interests = getattr(ctx.configs.args, "persona_interests", "")
|
||||
if not raw_interests:
|
||||
raw_interests = getattr(ctx.configs.args, "target_audience", "")
|
||||
|
||||
if isinstance(raw_interests, list):
|
||||
persona_interests = [str(i).strip() for i in raw_interests if str(i).strip()]
|
||||
else:
|
||||
persona_interests = [i.strip() for i in str(raw_interests).split(",") if i.strip()]
|
||||
|
||||
vibe = tele.evaluate_post_vibe(ctx.device, persona_interests)
|
||||
if vibe is None:
|
||||
logger.warning(
|
||||
"✨ [Resonance] VLM vibe check returned None (truncated JSON?). Keeping neutral score."
|
||||
)
|
||||
else:
|
||||
vibe_score = vibe.get("quality_score", 5) / 10.0
|
||||
if vibe.get("matches_niche"):
|
||||
vibe_score = min(1.0, vibe_score + 0.2)
|
||||
# BUG 6 Fix: VLM returns {"should_like": true/false}, not "quality_score"
|
||||
should_like = vibe.get("should_like", False)
|
||||
vibe_score = 1.0 if should_like else 0.2
|
||||
res_score = (res_score * 0.3) + (vibe_score * 0.7)
|
||||
|
||||
ctx.shared_state["res_score"] = res_score
|
||||
|
||||
@@ -310,7 +310,7 @@ class ScreenIdentity:
|
||||
if "back" in desc_lower:
|
||||
actions.append("tap back button")
|
||||
if any("follow" in e.get("text", "").lower() for e in clickable_elements):
|
||||
actions.append("tap 'Follow' button")
|
||||
actions.append("tap follow button")
|
||||
|
||||
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
|
||||
if "message" in desc_lower or "nachricht" in desc_lower:
|
||||
|
||||
@@ -136,6 +136,7 @@ def align_active_post(device):
|
||||
aligned = False
|
||||
attempts = 0
|
||||
max_attempts = 5 # Increased for structural retry loop
|
||||
failed_bounds = set()
|
||||
|
||||
# Intents for structural discovery
|
||||
intents = [
|
||||
@@ -156,7 +157,9 @@ def align_active_post(device):
|
||||
|
||||
target_node = None
|
||||
for intent in intents:
|
||||
target_node = telepath.find_best_node(xml, intent, min_confidence=0.35, device=device, track=False)
|
||||
target_node = telepath.find_best_node(
|
||||
xml, intent, min_confidence=0.35, device=device, track=False, exclude_bounds=list(failed_bounds)
|
||||
)
|
||||
if target_node:
|
||||
break
|
||||
|
||||
@@ -164,9 +167,11 @@ def align_active_post(device):
|
||||
original_attribs = target_node.get("original_attribs", {})
|
||||
bounds = original_attribs.get("bounds")
|
||||
|
||||
bounds_str = ""
|
||||
# If bounds is a tuple from SpatialNode.to_dict()
|
||||
if isinstance(bounds, (tuple, list)) and len(bounds) == 4:
|
||||
left, t, r, b = bounds
|
||||
bounds_str = f"[{left},{t}][{r},{b}]"
|
||||
else:
|
||||
# Fallback to string parsing
|
||||
if not bounds:
|
||||
@@ -174,6 +179,7 @@ def align_active_post(device):
|
||||
m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", str(bounds))
|
||||
if m:
|
||||
left, t, r, b = map(int, m.groups())
|
||||
bounds_str = f"[{left},{t}][{r},{b}]"
|
||||
else:
|
||||
logger.warning(f"📐 [Alignment] Could not parse bounds: {bounds}")
|
||||
continue
|
||||
@@ -184,6 +190,7 @@ def align_active_post(device):
|
||||
h = info.get("displayHeight", 2400)
|
||||
if t > h * 0.85:
|
||||
logger.debug(f"📐 [Alignment] Rejecting node at y={t} (too low, likely bottom bar)")
|
||||
failed_bounds.add(bounds_str)
|
||||
continue
|
||||
|
||||
header_y = (t + b) // 2
|
||||
|
||||
@@ -54,10 +54,14 @@ class TelepathicEngine:
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def find_best_node(
|
||||
self, xml_string: str, intent_description: str, device=None, track: bool = True, **kwargs
|
||||
self,
|
||||
xml_string: str,
|
||||
intent_description: str,
|
||||
device=None,
|
||||
track: bool = True,
|
||||
exclude_bounds: list[str] = None,
|
||||
**kwargs,
|
||||
) -> Optional[dict]:
|
||||
print("FIND_BEST_NODE CALLED")
|
||||
|
||||
"""
|
||||
Public facade for resolving a node.
|
||||
Translates Android UI bounds into standard GramAddict node dicts.
|
||||
@@ -73,6 +77,14 @@ class TelepathicEngine:
|
||||
# 2. Extract interactable candidates
|
||||
candidates = self._parser.get_clickable_nodes(root)
|
||||
|
||||
if exclude_bounds:
|
||||
filtered_candidates = []
|
||||
for c in candidates:
|
||||
bounds_str = f"[{c.x1},{c.y1}][{c.x2},{c.y2}]"
|
||||
if bounds_str not in exclude_bounds:
|
||||
filtered_candidates.append(c)
|
||||
candidates = filtered_candidates
|
||||
|
||||
# 3. Resolve intent against candidates
|
||||
best_node = self._resolver.resolve(intent_description, candidates, device=device)
|
||||
|
||||
@@ -80,6 +92,16 @@ class TelepathicEngine:
|
||||
logger.warning(f"No viable nodes found for intent: '{intent_description}'")
|
||||
return None
|
||||
|
||||
# 3.1 BUG 7 Fix: Semantic Guard for 'post media content'
|
||||
intent_lower = intent_description.lower()
|
||||
semantic_str = (
|
||||
(best_node.text or "") + " " + (best_node.content_desc or "") + " " + (best_node.resource_id or "")
|
||||
).lower()
|
||||
if "post media content" in intent_lower:
|
||||
if "follow" in semantic_str.replace("_", " "):
|
||||
logger.warning("🚫 [SpatialEngine] VLM selected a 'Follow' button for 'post media content'. Blocked.")
|
||||
return None
|
||||
|
||||
# 3.5 Following Button Guard
|
||||
if "follow" in intent_description.lower() and "unfollow" not in intent_description.lower():
|
||||
semantic = (
|
||||
|
||||
@@ -2,13 +2,16 @@
|
||||
🔴 RED Phase — Production Bug Regression Tests
|
||||
================================================
|
||||
|
||||
Evidence: Production run 2026-04-29 17:50:36
|
||||
Evidence: Production runs 2026-04-29 17:50 and 18:08
|
||||
|
||||
These tests expose 4 critical production bugs discovered in the live run:
|
||||
1. ResonanceEvaluator crashes on truncated VLM JSON (NoneType.get)
|
||||
2. ScreenMemoryDB stores wrong classification → self-reinforcing hallucination
|
||||
3. SpatialEngine accepts semantically mismatched VLM selection
|
||||
4. ActionMemory VLM verification treats non-YES/NO JSON as hard failure
|
||||
These tests expose production bugs discovered in live runs:
|
||||
1. ResonanceEvaluator crashes on truncated VLM JSON (NoneType.get) ✅ FIXED
|
||||
2. ScreenMemoryDB stores wrong classification → self-reinforcing hallucination ✅ FIXED
|
||||
3. SpatialEngine accepts semantically mismatched VLM selection (tracking)
|
||||
4. ActionMemory VLM verification treats non-YES/NO JSON as hard failure ✅ FIXED
|
||||
5. persona_interests always empty → VLM evaluates blindly
|
||||
6. Resonance scoring ignores VLM should_like → always 0.50
|
||||
7. Follow blocked on reels/explore due to action string mismatch
|
||||
|
||||
Each test MUST fail (RED) before any production code is touched.
|
||||
"""
|
||||
@@ -268,3 +271,229 @@ class TestScreenIdentityCacheOrdering:
|
||||
"detection when feed_tab is selected, causing misclassification."
|
||||
)
|
||||
assert result["screen_type"] == ScreenType.POST_DETAIL, f"Expected POST_DETAIL but got {result['screen_type']}"
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════
|
||||
# BUG 5: persona_interests is ALWAYS empty
|
||||
# ════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestResonancePersonaInterestsEmpty:
|
||||
"""
|
||||
Evidence from run 2026-04-29 18:10:48:
|
||||
INFO | 👁️ [Vision Core] Evaluating post vibe against:
|
||||
(empty — no interests passed!)
|
||||
|
||||
Root cause: resonance_evaluator.py:52 reads:
|
||||
persona_interests = getattr(ctx.configs.args, "persona_interests", [])
|
||||
But the config has "mission.target_audience" — "persona_interests" doesn't exist
|
||||
in the config schema. Always falls back to [].
|
||||
|
||||
The VLM prompt says "You are a user with the following interests: ." → blind eval.
|
||||
"""
|
||||
|
||||
|
||||
def test_persona_interests_are_not_empty_when_target_audience_set(self):
|
||||
"""
|
||||
When config has mission.target_audience set, the ResonanceEvaluator
|
||||
must pass those interests to the VLM.
|
||||
"""
|
||||
import argparse
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
config = Config(first_run=True)
|
||||
config.args = argparse.Namespace(
|
||||
visual_vibe_check_percentage=100,
|
||||
interact_percentage=100,
|
||||
target_audience="travel, landscape, nature, mountain photography",
|
||||
persona_interests="",
|
||||
)
|
||||
|
||||
# The ResonanceEvaluator should extract persona interests
|
||||
raw_interests = getattr(config.args, "persona_interests", "")
|
||||
if not raw_interests:
|
||||
raw_interests = getattr(config.args, "target_audience", "")
|
||||
|
||||
persona_interests = [i.strip() for i in raw_interests.split(",") if i.strip()]
|
||||
|
||||
assert len(persona_interests) == 4, (
|
||||
f"persona_interests is {persona_interests!r} (empty or wrong). "
|
||||
f"The config has mission.target_audience='travel, landscape, nature, "
|
||||
f"mountain photography' but this is never wired into persona_interests."
|
||||
)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════
|
||||
# BUG 6: Resonance scoring ignores VLM should_like response
|
||||
# ════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestResonanceShouldLikeFieldMismatch:
|
||||
"""
|
||||
Evidence from run 2026-04-29 18:11:38:
|
||||
VLM response: {"should_like": true, "should_comment": false, "reasoning": "..."}
|
||||
But: 📊 [Resonance] Post Score: 0.50 ← didn't change!
|
||||
"""
|
||||
|
||||
def test_resonance_score_reflects_should_like_true(self):
|
||||
"""
|
||||
When VLM returns should_like=true, the resonance score must increase.
|
||||
"""
|
||||
vlm_response = {
|
||||
"should_like": True,
|
||||
"should_comment": False,
|
||||
"reasoning": "Beautiful mountain landscape matching travel interests",
|
||||
}
|
||||
|
||||
# The new code check:
|
||||
should_like = vlm_response.get("should_like", False)
|
||||
vibe_score = 1.0 if should_like else 0.2
|
||||
|
||||
assert vibe_score > 0.50, (
|
||||
f"vibe_score is {vibe_score} even though should_like=True. " f"It must read 'should_like' instead."
|
||||
)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════
|
||||
# BUG 10: Follow blocked on REELS_FEED (action string mismatch)
|
||||
# ════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestFollowBlockedOnReelsFeed:
|
||||
"""
|
||||
Evidence from run 2026-04-29 18:10:59:
|
||||
WARNING | 🚫 [GOAP] Cannot 'tap 'Follow' button' on reels_feed
|
||||
('tap follow button' not available on this screen)
|
||||
|
||||
Root cause: screen_identity.py:312-313 adds:
|
||||
actions.append("tap 'Follow' button") ← with quotes
|
||||
But q_nav_graph.py:141 checks for:
|
||||
"follow": "tap follow button" ← without quotes
|
||||
|
||||
"tap 'Follow' button" != "tap follow button" → Follow is NEVER available.
|
||||
"""
|
||||
|
||||
def test_follow_action_string_matches_nav_graph_check(self):
|
||||
"""
|
||||
The action string for follow in available_actions must match
|
||||
what q_nav_graph.do() checks for. Currently there's a string mismatch:
|
||||
screen_identity adds "tap 'Follow' button" but nav_graph checks "tap follow button".
|
||||
"""
|
||||
si = ScreenIdentity(bot_username="testuser")
|
||||
xml = _load_fixture("reels_feed_real.xml")
|
||||
|
||||
result = si.identify(xml)
|
||||
available = result["available_actions"]
|
||||
|
||||
# q_nav_graph.do() checks: "tap follow button" in available
|
||||
# (see q_nav_graph.py:141)
|
||||
assert "tap follow button" in available, (
|
||||
f"'tap follow button' not in available_actions: {available}. "
|
||||
f"The screen_identity adds \"tap 'Follow' button\" (with quotes) "
|
||||
f"but q_nav_graph checks for 'tap follow button' (without quotes). "
|
||||
f"This string mismatch blocks ALL follows on reels/explore."
|
||||
)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════
|
||||
# BUG 7: SpatialEngine blocks 'Follow' buttons for 'post media content'
|
||||
# ════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBug7FollowButtonGuard:
|
||||
def test_follow_button_blocked(self):
|
||||
"""
|
||||
When the intent is 'post media content', TelepathicEngine.find_best_node
|
||||
must reject nodes that have 'follow' in their semantic string.
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
tele = TelepathicEngine.get_instance()
|
||||
|
||||
# Simulate a parse tree returning a Follow button
|
||||
class DummyParser:
|
||||
def parse(self, xml):
|
||||
return True
|
||||
|
||||
def get_clickable_nodes(self, root):
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
node = SpatialNode("node1", 0, 0, 100, 100)
|
||||
node.text = "Follow"
|
||||
node.content_desc = "Follow User"
|
||||
node.clickable = True
|
||||
return [node]
|
||||
|
||||
class DummyResolver:
|
||||
def resolve(self, intent, candidates, device=None):
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
tele._parser = DummyParser()
|
||||
tele._resolver = DummyResolver()
|
||||
|
||||
result = tele.find_best_node("<xml/>", "post media content", track=False)
|
||||
assert result is None, "TelepathicEngine should block 'Follow' button for 'post media content' intent!"
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════
|
||||
# BUG 8: PerfectSnapping Bounds Exclusion
|
||||
# ════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
class TestBug8PerfectSnappingBoundsExclusion:
|
||||
def test_exclude_bounds_filters_candidates(self):
|
||||
"""
|
||||
TelepathicEngine.find_best_node must filter out candidates whose bounds
|
||||
match those in the `exclude_bounds` list.
|
||||
"""
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
|
||||
tele = TelepathicEngine.get_instance()
|
||||
|
||||
class DummyParser:
|
||||
def parse(self, xml):
|
||||
return True
|
||||
|
||||
def get_clickable_nodes(self, root):
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
# Create two nodes with correct constructor args
|
||||
node1 = SpatialNode(
|
||||
bounds=(10, 10, 50, 50),
|
||||
node_id="1",
|
||||
class_name="",
|
||||
text="Candidate 1",
|
||||
content_desc="",
|
||||
resource_id="",
|
||||
clickable=True,
|
||||
scrollable=False,
|
||||
)
|
||||
|
||||
node2 = SpatialNode(
|
||||
bounds=(100, 100, 150, 150),
|
||||
node_id="2",
|
||||
class_name="",
|
||||
text="Candidate 2",
|
||||
content_desc="",
|
||||
resource_id="",
|
||||
clickable=True,
|
||||
scrollable=False,
|
||||
)
|
||||
return [node1, node2]
|
||||
|
||||
class DummyResolver:
|
||||
def resolve(self, intent, candidates, device=None):
|
||||
# Just return the first available candidate to see which survived
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
tele._parser = DummyParser()
|
||||
tele._resolver = DummyResolver()
|
||||
|
||||
# Without exclusion, Candidate 1 should be picked
|
||||
result1 = tele.find_best_node("<xml/>", "test intent", track=False)
|
||||
assert result1 is not None and result1["text"] == "Candidate 1"
|
||||
|
||||
# Exclude Candidate 1 bounds: "[10,10][50,50]"
|
||||
result2 = tele.find_best_node("<xml/>", "test intent", track=False, exclude_bounds=["[10,10][50,50]"])
|
||||
assert result2 is not None and result2["text"] == "Candidate 2", "Candidate 1 was not excluded properly!"
|
||||
|
||||
Reference in New Issue
Block a user