feat: structural test integrity enforcement — mock ban, brain contract tests, UI change noise threshold

- Add permanent mock ban guard in root conftest.py that fails any test
  importing unittest.mock at COLLECTION TIME (before execution)
- Add 8 brain output contract tests reproducing the exact production bug:
  LLM thinks 'press back' but parser extracts 'tap messages tab' from
  the <think> block
- Add UI change noise threshold (MIN_UI_CHANGE_BYTES=50) to prevent
  false-positive 'ui_changed' from 1-byte XML diffs (timestamps/whitespace)
- Verify planner correctly strips masked actions from Brain prompt
This commit is contained in:
2026-04-28 23:45:22 +02:00
parent 5fcf1f180b
commit ad012b4cd4
3 changed files with 282 additions and 2 deletions

View File

@@ -356,9 +356,16 @@ class GoalExecutor:
# Determine if this was a navigation or an interaction
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"])
action_success = False
ui_changed = post_xml != xml_dump
# ── UI Change Detection with Noise Threshold ──
# Raw string diffs of < 50 bytes are noise (timestamps, whitespace, counters).
# A real navigation changes the XML by hundreds/thousands of bytes.
MIN_UI_CHANGE_BYTES = 50
xml_delta = abs(len(post_xml) - len(xml_dump))
ui_changed = post_xml != xml_dump and xml_delta >= MIN_UI_CHANGE_BYTES
logger.debug(
f"[GOAP Verify] ui_changed={ui_changed}, " f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}"
f"[GOAP Verify] ui_changed={ui_changed}, "
f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}, delta={xml_delta}b"
)
if is_navigation: