mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

test(e2e): decompose monolithic test suite and fortify semantic guards

Browse Source
This commit is contained in:
Marc Mintel
2026-04-28 23:09:04 +02:00
parent e94dfe8c5c
commit dc4b576bc1
19 changed files with 458 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
GramAddict/core/llm_provider.py
View File

@@ -344,16 +344,20 @@ def query_llm(
return {"response": content}
else:
# Ollama returns response OR thinking (for reasoning models)
content = resp_json.get("response") or resp_json.get("thinking") or ""
raw_response = resp_json.get("response", "")
raw_thinking = resp_json.get("thinking", "")
logger.debug(f"DEBUG LLM PAYLOAD: response='{raw_response}', thinking='{raw_thinking}'")
content = raw_response or raw_thinking or ""
if format_json:
extracted = extract_json(content)
if not extracted:
# Log more context if JSON extraction fails
logger.debug(f"Ollama raw content (for JSON extraction): {content[:200]}...")
raise ValueError("Ollama returned non-JSON content when JSON was expected.")
resp_json["response"] = extracted
logger.warning(f"Failed to extract JSON from content: {content[:100]}")
else:
content = extracted
return resp_json
return {"response": content}
except requests.exceptions.ConnectionError:
logger.error(f"⚠️ [LLM Provider] Connection refused for {model} at {url}. Is the service running?")
except Exception as e:

9
GramAddict/core/navigation/brain.py
View File

@@ -36,7 +36,7 @@ def ask_brain_for_action(
"INSTRUCTIONS:\n"
"1. Reason about where you are. Consider the screen type and what actions make sense on that screen.\n"
"2. If the goal requires navigating away from the current screen, choose the action that moves you closest to the goal.\n"
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). Some screens like stories or modals are NOT scrollable.\n"
"3. 'scroll down' reveals more UI elements on scrollable screens (feeds, profiles, lists). If your target is likely on this screen but not currently visible, you MUST choose 'scroll down'.\n"
"4. 'press back' exits the current screen and returns to the previous one. Use it when you are on a screen that doesn't lead to your goal.\n"
"5. DO NOT hallucinate actions. Reply ONLY with the exact string from the available actions list.\n"
"6. Reply with ONLY the action string, nothing else."
@@ -44,7 +44,12 @@ def ask_brain_for_action(
try:
response = query_llm(
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False, max_tokens=20
url=url,
model=model,
prompt="Choose the next best action.",
system=prompt,
format_json=False,
max_tokens=250,
)
if response:
result = response if isinstance(response, str) else response.get("response", "")

41
GramAddict/core/perception/action_memory.py
View File

@@ -208,14 +208,43 @@ class ActionMemory:
logger.debug(f"🧠 [ActionMemory] Structural delta detected for toggle '{intent}'. Verification PASS.")
return True
else:
# If the intent is an abstract goal (like "find customers"), diff > 50 is NOT enough.
# We must force visual VLM confirmation because clicking the wrong thing (like "Create highlight")
# also produces a large diff but achieves the wrong goal.
if diff > 50:
logger.debug(
f"🧠 [ActionMemory] Structural change detected for navigation '{intent}'. Verification PASS."
)
return True
# Is it a standard structural transition?
from GramAddict.core.screen_topology import ScreenTopology
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
return False
# We don't have screen type here, so we just check if it's in the HD Map keys
is_standard = any(intent in transitions for transitions in ScreenTopology.TRANSITIONS.values())
if is_standard:
logger.debug(
f"🧠 [ActionMemory] Structural change detected for known navigation '{intent}'. Verification PASS."
)
return True
else:
logger.info(
f"👁️ [ActionMemory] Abstract intent '{intent}' caused UI change. Forcing VLM visual verification..."
)
# For abstract intents, we must visually verify if it actually helped!
# If device is available, we use VLM. If not, we fail safe.
if device:
from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
evaluator = SemanticEvaluator()
prompt = f"The user just attempted to perform the action: '{intent}'. Does the current screen match the expected outcome? Answer ONLY with the word YES or NO."
try:
response = evaluator._query_vlm(prompt, device.get_screenshot_b64())
if response and "yes" in response.lower() and "no" not in response.lower():
return True
else:
logger.warning(f"⚠️ [ActionMemory] VLM rejected success for abstract intent '{intent}'.")
return False
except Exception as e:
logger.error(f"VLM visual verification failed: {e}")
logger.warning(f"⚠️ [ActionMemory] Cannot visually verify abstract intent '{intent}'. Failing safe.")
return False
def _intent_matches_node(intent: str, semantic_string: str) -> bool:

72
GramAddict/core/perception/intent_resolver.py
View File

@@ -8,8 +8,6 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
logger = logging.getLogger(__name__)
# Navigation tab intent → resource_id keyword mapping
# These are STRUCTURAL guards (bottom 15% zone), not string-matching heuristics.
_NAV_TAB_MAP = {
"tap home tab": "feed_tab",
"tap explore tab": "search_tab",
@@ -19,6 +17,18 @@ _NAV_TAB_MAP = {
}
def _humanize_desc(desc: str) -> str:
"""
Inserts a space between numbers and letters to fix Instagram's concatenated content-desc.
Example: "991following" -> "991 following", "140Kfollowers" -> "140K followers"
"""
if not desc:
return ""
import re
return re.sub(r"(\d[KMBkmb]?)([a-z])", r"\1 \2", desc)
class IntentResolver:
"""
Vision-First Intent Resolver.
@@ -39,7 +49,7 @@ class IntentResolver:
# ──────────────────────────────────────────────
def resolve(
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400, device=None
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
) -> Optional[SpatialNode]:
if not candidates:
return None
@@ -72,9 +82,17 @@ class IntentResolver:
if intent_lower in abstract_goals:
return None
# --- Strict VLM Hallucination Guard ---
# For known structural targets that the VLM frequently hallucinates when they are missing,
# we enforce a strict failure if they weren't caught by the structural fast paths.
# ── PRIMARY PATH: Visual Discovery ──
# If we have a device, the VLM SEES the screen and decides.
if device is not None and (
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
):
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
return self._visual_discovery(intent_description, candidates, device)
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
# we enforce a strict failure.
if "following list" in intent_lower or "followers list" in intent_lower or "tap message button" in intent_lower:
logger.warning(
f"🛡️ [Hallucination Guard] Intent '{intent_description}' is a strict structural target. "
@@ -82,14 +100,6 @@ class IntentResolver:
)
return None
# ── PRIMARY PATH: Visual Discovery ──
# If we have a device, the VLM SEES the screen and decides.
if device:
result = self._visual_discovery(intent_description, candidates, device)
if result:
return result
logger.warning(f"👁️ [Visual Discovery] No match found for '{intent_description}', trying text fallback.")
# ── FALLBACK: Text-based VLM resolution ──
# Only used when device is unavailable (e.g., unit tests without screenshots).
return self._text_based_resolve(intent_description, candidates, device)
@@ -112,15 +122,8 @@ class IntentResolver:
img = device.deviceV2.screenshot()
# Stage 1: Basic area filter + exclude system UI and notifications
pre_filtered = [
n
for n in candidates
if 200 < n.area < 400000
and "com.android.systemui" not in (n.resource_id or "")
and "notification:" not in (n.content_desc or "").lower()
and "per cent" not in (n.content_desc or "").lower()
]
# Stage 1: Basic area filter + exclude system UI and notifications (ALREADY HANDLED in _visual_discovery)
pre_filtered = candidates
# Stage 2: Spatial deduplication
# A node could completely contain another.
@@ -241,6 +244,16 @@ class IntentResolver:
from GramAddict.core.config import Config
from GramAddict.core.llm_provider import query_telepathic_llm
# Pre-filter candidates by area and system UI before any semantic matching
candidates = [
n
for n in candidates
if 200 < n.area < 400000
and "com.android.systemui" not in (n.resource_id or "")
and "notification:" not in (n.content_desc or "").lower()
and "per cent" not in (n.content_desc or "").lower()
]
# --- Strict Button Guard ---
# If the intent specifically asks for a "button", "icon", or "tab",
# filter out candidates that contain long text (e.g. captions, comments)
@@ -308,9 +321,11 @@ class IntentResolver:
node = box_map[idx]
label_parts = []
if node.content_desc:
label_parts.append(f"desc='{node.content_desc[:50]}'")
desc = _humanize_desc(node.content_desc)
label_parts.append(f"desc='{desc[:50]}'")
if node.text and node.text != node.content_desc:
label_parts.append(f"text='{node.text[:50]}'")
text = _humanize_desc(node.text)
label_parts.append(f"text='{text[:50]}'")
if not label_parts:
label_parts.append("(no visible text)")
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
@@ -330,7 +345,8 @@ class IntentResolver:
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
f"5. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
f"6. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
)
@@ -394,8 +410,8 @@ class IntentResolver:
node_context = []
for i, node in enumerate(filtered_candidates):
text = node.text or ""
desc = node.content_desc or ""
text = _humanize_desc(node.text or "")
desc = _humanize_desc(node.content_desc or "")
res_id = node.resource_id or ""
node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]")

1
GramAddict/core/perception/screen_identity.py
View File

@@ -321,6 +321,7 @@ class ScreenIdentity:
# Scroll
actions.append("scroll down")
actions.append("scroll up")
actions.append("press back")
return list(set(actions)) # Deduplicate