test(e2e): purge deceptive pytest.skip masks hiding VLM failures

This commit is contained in:
2026-04-28 21:49:31 +02:00
parent 7aa6bfccf6
commit e94dfe8c5c
5 changed files with 26 additions and 20 deletions

View File

@@ -15,7 +15,11 @@ def ask_brain_for_action(
return None
cfg = Config()
url = getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate") if hasattr(cfg, "args") else "http://localhost:11434/api/generate"
url = (
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
if hasattr(cfg, "args")
else "http://localhost:11434/api/generate"
)
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
prompt = (
@@ -40,7 +44,7 @@ def ask_brain_for_action(
try:
response = query_llm(
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False, max_tokens=20
)
if response:
result = response if isinstance(response, str) else response.get("response", "")

View File

@@ -34,8 +34,10 @@ def test_brain_recommends_scroll_when_trapped():
logger.info(f"Brain action returned: '{brain_action}'")
if brain_action is None or brain_action == "":
pytest.skip("Brain LLM returned None or empty string. Ollama timeout or hallucination.")
assert (
brain_action is not None and brain_action != ""
), "Brain LLM returned None or empty string. Ollama timeout or hallucination."
if brain_action != "scroll down":
pytest.skip(f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky.")
assert (
brain_action == "scroll down"
), f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky."

View File

@@ -320,11 +320,10 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
selected_id = (selected_node.resource_id or "").lower()
# THE CRITICAL ASSERTION: Must be "following", NOT "followers"
if "following" not in selected_id and "following" not in selected_desc and "following" not in selected_text:
pytest.skip(
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
f"Skipping because small local VLMs often fail this negative constraint."
)
assert "following" in selected_id or "following" in selected_desc or "following" in selected_text, (
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
f"This proves the local VLM failed the negative constraint."
)
assert (
"followers" not in selected_id
), f"VLM CONFUSED followers with following! Selected: id='{selected_node.resource_id}'"

View File

@@ -98,8 +98,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
return True
return False
if not _node_has_marker(result, "comment"):
pytest.skip(
f"VLM picked WRONG element for 'tap comment button'!\n"
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
)
assert _node_has_marker(result, "comment"), (
f"VLM picked WRONG element for 'tap comment button'!\n"
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
)

View File

@@ -120,10 +120,12 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
selected_id = (result.resource_id or "").lower()
selected_desc = (result.content_desc or "").lower()
if "following" not in selected_id and "following" not in selected_desc:
pytest.skip(f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'")
if "followers" in selected_id:
pytest.skip(f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'")
assert (
"following" in selected_id or "following" in selected_desc
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
assert (
"followers" not in selected_id
), f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'"
# ═══════════════════════════════════════════════════════