From f1a8573be8791c8315d7a10dc06b6f62f54ea38c Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Fri, 1 May 2026 16:55:53 +0200 Subject: [PATCH] test(e2e): eliminate lying tests via behavioral truth assertions --- .gitignore | 3 + tests/e2e/fixtures/carousel_post_dump.xml | 193 ++++++++++++++++++++++ tests/e2e/test_workflow_dm_inbox.py | 22 +-- tests/e2e/test_workflow_post_detail.py | 32 ++-- tests/e2e/test_workflow_profiles.py | 30 ++++ tests/e2e/test_workflow_search_feed.py | 14 +- tests/e2e/test_workflow_stories_feed.py | 4 + 7 files changed, 261 insertions(+), 37 deletions(-) create mode 100644 tests/e2e/fixtures/carousel_post_dump.xml diff --git a/.gitignore b/.gitignore index 461efbe..26284dc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ !tests/fixtures/*.xml !tests/fixtures/*.jpg !tests/fixtures/*.json +!tests/e2e/fixtures/*.xml +!tests/e2e/fixtures/*.jpg +!tests/e2e/fixtures/*.json logs/ *.pyc __pycache__/ diff --git a/tests/e2e/fixtures/carousel_post_dump.xml b/tests/e2e/fixtures/carousel_post_dump.xml new file mode 100644 index 0000000..90eafc1 --- /dev/null +++ b/tests/e2e/fixtures/carousel_post_dump.xml @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/e2e/test_workflow_dm_inbox.py b/tests/e2e/test_workflow_dm_inbox.py index ecbf849..9f16d7e 100644 --- a/tests/e2e/test_workflow_dm_inbox.py +++ b/tests/e2e/test_workflow_dm_inbox.py @@ -11,8 +11,6 @@ import os from tests.e2e.conftest import E2EDeviceStub -from GramAddict.core.situational_awareness import SituationType - FIXTURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") @@ -30,13 +28,13 @@ def test_dm_inbox_processes_without_crashes(monkeypatch, e2e_workflow_ctx): results, ctx = e2e_workflow_ctx(device) executed_count = sum(1 for r in results if r.executed) - assert executed_count >= 1, ( - f"Only {executed_count} plugin(s) executed on DM inbox." - ) + assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on DM inbox." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the DM inbox!" - assert "back" not in device.pressed_keys, ( - "obstacle_guard false positive on DM inbox!" - ) + assert "back" not in device.pressed_keys, "obstacle_guard false positive on DM inbox!" def test_dm_thread_processes_without_crashes(monkeypatch, e2e_workflow_ctx): @@ -48,6 +46,8 @@ def test_dm_thread_processes_without_crashes(monkeypatch, e2e_workflow_ctx): results, ctx = e2e_workflow_ctx(device) executed_count = sum(1 for r in results if r.executed) - assert executed_count >= 1, ( - f"Only {executed_count} plugin(s) executed on DM thread." - ) + assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on DM thread." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the DM thread!" diff --git a/tests/e2e/test_workflow_post_detail.py b/tests/e2e/test_workflow_post_detail.py index 2253dbb..cb16f16 100644 --- a/tests/e2e/test_workflow_post_detail.py +++ b/tests/e2e/test_workflow_post_detail.py @@ -11,9 +11,7 @@ import os from tests.e2e.conftest import E2EDeviceStub -from GramAddict.core.situational_awareness import SituationType - -FIXTURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") def _load_fixture(name): @@ -25,20 +23,18 @@ def test_post_detail_processes_without_crashes(monkeypatch, e2e_workflow_ctx): """ A post detail page must be processable without crashes. """ - # Use the more realistic fixture if available - fixture_name = "home_feed_with_ad.xml" - post_xml = _load_fixture(fixture_name) + post_xml = _load_fixture("post_detail_real.xml") device = E2EDeviceStub([post_xml, post_xml]) results, ctx = e2e_workflow_ctx(device) executed_count = sum(1 for r in results if r.executed) - assert executed_count >= 1, ( - f"Only {executed_count} plugin(s) executed on post detail." - ) + assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on post detail." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the post!" - assert "back" not in device.pressed_keys, ( - "obstacle_guard false positive on post detail!" - ) + assert "back" not in device.pressed_keys, "obstacle_guard false positive on post detail!" def test_carousel_post_processes_without_crashes(monkeypatch, e2e_workflow_ctx): @@ -50,10 +46,10 @@ def test_carousel_post_processes_without_crashes(monkeypatch, e2e_workflow_ctx): results, ctx = e2e_workflow_ctx(device) executed_count = sum(1 for r in results if r.executed) - assert executed_count >= 1, ( - f"Only {executed_count} plugin(s) executed on carousel post." - ) + assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on carousel post." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the carousel!" - assert "back" not in device.pressed_keys, ( - "obstacle_guard false positive on carousel post!" - ) + assert "back" not in device.pressed_keys, "obstacle_guard false positive on carousel post!" diff --git a/tests/e2e/test_workflow_profiles.py b/tests/e2e/test_workflow_profiles.py index 050e742..2fdc5d2 100644 --- a/tests/e2e/test_workflow_profiles.py +++ b/tests/e2e/test_workflow_profiles.py @@ -12,6 +12,8 @@ Real: Cognitive stack, PluginRegistry, SessionState, Config. import os +import pytest + from tests.e2e.conftest import E2EDeviceStub FIXTURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") @@ -40,6 +42,14 @@ def test_user_profile_processes_without_crashes(monkeypatch, e2e_workflow_ctx): assert "back" not in device.pressed_keys, "obstacle_guard false positive on user profile!" +@pytest.mark.xfail( + strict=True, + reason=( + "BUG/FIXTURE: scraping_profile_dump.xml is identified as 'own_profile'. " + "The default GOAP pipeline correctly refuses to 'follow' or 'like' oneself, " + "leading to 0 interactions. Test needs a specific scrape intent or a different fixture." + ), +) def test_scraping_profile_processes_without_crashes(monkeypatch, e2e_workflow_ctx): """ The scraping profile dump must be processable without crashes. @@ -50,6 +60,10 @@ def test_scraping_profile_processes_without_crashes(monkeypatch, e2e_workflow_ct executed_count = sum(1 for r in results if r.executed) assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on scraping profile." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the scraping profile!" def test_followers_list_processes_without_crashes(monkeypatch, e2e_workflow_ctx): @@ -62,8 +76,20 @@ def test_followers_list_processes_without_crashes(monkeypatch, e2e_workflow_ctx) executed_count = sum(1 for r in results if r.executed) assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on followers list." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the followers list!" +@pytest.mark.xfail( + strict=True, + reason=( + "BUG: GOAP searches for 'tap Follow button' but unfollow_list XML only has " + "'Following' buttons. The Semantic Guard filters them out → zero interactions. " + "Fix: screen_identity must map follow_list to 'tap Following button' action." + ), +) def test_unfollow_list_processes_without_crashes(monkeypatch, e2e_workflow_ctx): """ The unfollow list must be processable without crashes. @@ -74,3 +100,7 @@ def test_unfollow_list_processes_without_crashes(monkeypatch, e2e_workflow_ctx): executed_count = sum(1 for r in results if r.executed) assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on unfollow list." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the unfollow list!" diff --git a/tests/e2e/test_workflow_search_feed.py b/tests/e2e/test_workflow_search_feed.py index ccf83fe..e6fc7b9 100644 --- a/tests/e2e/test_workflow_search_feed.py +++ b/tests/e2e/test_workflow_search_feed.py @@ -11,8 +11,6 @@ import os from tests.e2e.conftest import E2EDeviceStub -from GramAddict.core.situational_awareness import SituationType - FIXTURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") @@ -30,10 +28,10 @@ def test_search_feed_processes_without_crashes(monkeypatch, e2e_workflow_ctx): results, ctx = e2e_workflow_ctx(device) executed_count = sum(1 for r in results if r.executed) - assert executed_count >= 1, ( - f"Only {executed_count} plugin(s) executed on Search feed." - ) + assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on Search feed." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the Search feed!" - assert "back" not in device.pressed_keys, ( - "obstacle_guard false positive on Search feed!" - ) + assert "back" not in device.pressed_keys, "obstacle_guard false positive on Search feed!" diff --git a/tests/e2e/test_workflow_stories_feed.py b/tests/e2e/test_workflow_stories_feed.py index 865b978..6b4cad8 100644 --- a/tests/e2e/test_workflow_stories_feed.py +++ b/tests/e2e/test_workflow_stories_feed.py @@ -47,3 +47,7 @@ def test_story_view_full_processes_without_crashes(monkeypatch, e2e_workflow_ctx executed_count = sum(1 for r in results if r.executed) assert executed_count >= 1, f"Only {executed_count} plugin(s) executed on Story view." + # 🚨 BEHAVIORAL TRUTH ASSERTION 🚨 + assert ( + len(device.clicks) > 0 or len(device.swipes) > 0 + ), "LIE DETECTED: The pipeline claimed success but did not interact with the Story view!"