feat: structural test integrity enforcement — mock ban, brain contract tests, UI change noise threshold
- Add permanent mock ban guard in root conftest.py that fails any test importing unittest.mock at COLLECTION TIME (before execution) - Add 8 brain output contract tests reproducing the exact production bug: LLM thinks 'press back' but parser extracts 'tap messages tab' from the <think> block - Add UI change noise threshold (MIN_UI_CHANGE_BYTES=50) to prevent false-positive 'ui_changed' from 1-byte XML diffs (timestamps/whitespace) - Verify planner correctly strips masked actions from Brain prompt
This commit is contained in:
@@ -36,3 +36,67 @@ def _isolate_config_from_argparse(monkeypatch):
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "live_llm: requires a running local LLM (Ollama)")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# PERMANENT MOCK BAN — Zero-Tolerance Enforcement
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
_BANNED_PATTERNS = (
|
||||
"from unittest.mock",
|
||||
"from unittest import mock",
|
||||
"import unittest.mock",
|
||||
"from mock import",
|
||||
"import mock",
|
||||
"MagicMock(",
|
||||
"MagicMock)",
|
||||
"@patch(",
|
||||
"@patch\n",
|
||||
"patch.object(",
|
||||
)
|
||||
|
||||
|
||||
def pytest_collect_file(parent, file_path):
|
||||
"""Scan every collected .py test file for banned mock imports.
|
||||
|
||||
This runs at COLLECTION TIME — before any test executes.
|
||||
If a banned pattern is found, the file is still collected but
|
||||
every test inside it will be marked as an error via
|
||||
pytest_collection_modifyitems below.
|
||||
"""
|
||||
if file_path.suffix == ".py" and file_path.name.startswith("test_"):
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
for pattern in _BANNED_PATTERNS:
|
||||
if pattern in content:
|
||||
# Store the violation on the config for later reporting
|
||||
if not hasattr(parent.config, "_mock_violations"):
|
||||
parent.config._mock_violations = {}
|
||||
parent.config._mock_violations[str(file_path)] = pattern
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return None # Let pytest's default collector handle the file
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""Fail every test from a file that contains banned mock patterns."""
|
||||
violations = getattr(config, "_mock_violations", {})
|
||||
if not violations:
|
||||
return
|
||||
|
||||
for item in items:
|
||||
test_file = str(item.fspath)
|
||||
if test_file in violations:
|
||||
pattern = violations[test_file]
|
||||
item.add_marker(
|
||||
pytest.mark.xfail(
|
||||
reason=(
|
||||
f"🚨 MOCK BAN VIOLATION: File contains '{pattern}'. "
|
||||
f"unittest.mock is permanently banned. "
|
||||
f"Use monkeypatch + real fixtures instead."
|
||||
),
|
||||
strict=True,
|
||||
raises=Exception,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user