mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

test(e2e): Fix LLM mock hallucinations to use structurally merged search string for 100% truthful navigation tests

Browse Source
This commit is contained in:
Marc Mintel
2026-05-01 22:27:38 +02:00
parent 2e1edec56a
commit aa5184786e
2 changed files with 36 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
tests/e2e/conftest.py
View File

@@ -743,6 +743,9 @@ def mock_llm_network_calls(monkeypatch, request):
match = re.search(r"\[(\d+)\][^\[]*row_feed_photo_profile", prompt.lower())
if match:
return json.dumps({"selected_index": int(match.group(1))})
match = re.search(r"\[(\d+)\][^\[]*row_search_user_container", prompt.lower())
if match:
return json.dumps({"selected_index": int(match.group(1))})
if "follow" in intent:
match = re.search(r"\[(\d+)\][^\[]*follow", prompt.lower())
if match:
@@ -751,10 +754,32 @@ def mock_llm_network_calls(monkeypatch, request):
return json.dumps({"selected_index": 0})
if "Find the exact box number" in prompt:
if "profile tab" in intent:
match = re.search(r"\[(\d+)\][^\n]*?desc='profile'", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
if "tab" in intent:
# 🚨 VLM HALLUCINATION SIMULATION 🚨
# The real VLM often gets confused and picks a user profile or text instead of the bottom tab.
# If our intent_resolver didn't filter out non-tab items properly, the prompt will contain them.
# Let's actively try to pick a bad node first (e.g. 'desc='marisaundmarc'' or any non-tab keyword)
match_bad = re.search(r"\[(\d+)\][^\n]*?desc='marisaundmarc'", prompt.lower())
if match_bad:
return json.dumps({"box": int(match_bad.group(1))})
# If the bad node was successfully filtered, pick the correct tab
if "profile" in intent:
match = re.search(r"\[(\d+)\][^\n]*?desc='profile'", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
elif "explore" in intent or "search" in intent:
match = re.search(r"\[(\d+)\][^\n]*?search and explore", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
elif "reels" in intent:
match = re.search(r"\[(\d+)\][^\n]*?reels", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
elif "home" in intent:
match = re.search(r"\[(\d+)\][^\n]*?home", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
if "following" in intent:
match = re.search(r"\[(\d+)\][^\n]*?following", prompt.lower())
if match:
@@ -794,6 +819,10 @@ def mock_llm_network_calls(monkeypatch, request):
match = re.search(r"\[(\d+)\][^\n]*?row_feed_photo_profile", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
# Fallback for search feed usernames (since resource_id is stripped, we match the merged text)
match = re.search(r"\[(\d+)\][^\n]*?lokmatas", prompt.lower())
if match:
return json.dumps({"box": int(match.group(1))})
if "follow" in intent:
match = re.search(r"\[(\d+)\][^\n]*?follow", prompt.lower())
if match:

4
tests/e2e/test_workflow_search_feed.py
View File

@@ -19,10 +19,12 @@ def _load_fixture(name):
return f.read()
def test_search_feed_processes_without_crashes(monkeypatch, e2e_workflow_ctx):
def test_search_feed_processes_without_crashes(e2e_configs, e2e_workflow_ctx):
"""
The Search feed must be processable without crashes.
"""
e2e_configs.args.profile_visit_percentage = 100
search_xml = _load_fixture("search_feed_dump.xml")
device = E2EDeviceStub([search_xml, search_xml])
results, ctx = e2e_workflow_ctx(device)