test(e2e): purge deceptive pytest.skip masks hiding VLM failures
This commit is contained in:
@@ -15,7 +15,11 @@ def ask_brain_for_action(
|
||||
return None
|
||||
|
||||
cfg = Config()
|
||||
url = getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate") if hasattr(cfg, "args") else "http://localhost:11434/api/generate"
|
||||
url = (
|
||||
getattr(cfg.args, "ai_model_url", "http://localhost:11434/api/generate")
|
||||
if hasattr(cfg, "args")
|
||||
else "http://localhost:11434/api/generate"
|
||||
)
|
||||
model = getattr(cfg.args, "ai_model", "qwen3.5:latest") if hasattr(cfg, "args") else "qwen3.5:latest"
|
||||
|
||||
prompt = (
|
||||
@@ -40,7 +44,7 @@ def ask_brain_for_action(
|
||||
|
||||
try:
|
||||
response = query_llm(
|
||||
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False
|
||||
url=url, model=model, prompt="Choose the next best action.", system=prompt, format_json=False, max_tokens=20
|
||||
)
|
||||
if response:
|
||||
result = response if isinstance(response, str) else response.get("response", "")
|
||||
|
||||
@@ -34,8 +34,10 @@ def test_brain_recommends_scroll_when_trapped():
|
||||
|
||||
logger.info(f"Brain action returned: '{brain_action}'")
|
||||
|
||||
if brain_action is None or brain_action == "":
|
||||
pytest.skip("Brain LLM returned None or empty string. Ollama timeout or hallucination.")
|
||||
assert (
|
||||
brain_action is not None and brain_action != ""
|
||||
), "Brain LLM returned None or empty string. Ollama timeout or hallucination."
|
||||
|
||||
if brain_action != "scroll down":
|
||||
pytest.skip(f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky.")
|
||||
assert (
|
||||
brain_action == "scroll down"
|
||||
), f"VLM chose '{brain_action}' instead of 'scroll down'. Small local models can be flaky."
|
||||
|
||||
@@ -320,11 +320,10 @@ def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
|
||||
selected_id = (selected_node.resource_id or "").lower()
|
||||
|
||||
# THE CRITICAL ASSERTION: Must be "following", NOT "followers"
|
||||
if "following" not in selected_id and "following" not in selected_desc and "following" not in selected_text:
|
||||
pytest.skip(
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"Skipping because small local VLMs often fail this negative constraint."
|
||||
)
|
||||
assert "following" in selected_id or "following" in selected_desc or "following" in selected_text, (
|
||||
f"VLM hallucinated and selected wrong node! Got: desc='{selected_node.content_desc}', text='{selected_node.text}', id='{selected_node.resource_id}'. "
|
||||
f"This proves the local VLM failed the negative constraint."
|
||||
)
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"VLM CONFUSED followers with following! Selected: id='{selected_node.resource_id}'"
|
||||
|
||||
@@ -98,8 +98,7 @@ def test_home_feed_comment_button_extraction(make_real_device_with_image):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not _node_has_marker(result, "comment"):
|
||||
pytest.skip(
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
assert _node_has_marker(result, "comment"), (
|
||||
f"VLM picked WRONG element for 'tap comment button'!\n"
|
||||
f" Selected: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
)
|
||||
|
||||
@@ -120,10 +120,12 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
selected_desc = (result.content_desc or "").lower()
|
||||
|
||||
if "following" not in selected_id and "following" not in selected_desc:
|
||||
pytest.skip(f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'")
|
||||
if "followers" in selected_id:
|
||||
pytest.skip(f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'")
|
||||
assert (
|
||||
"following" in selected_id or "following" in selected_desc
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
assert (
|
||||
"followers" not in selected_id
|
||||
), f"Visual discovery CONFUSED followers with following! Selected: id='{result.resource_id}'"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user