fix(sae): stabilize navigation engine, fix container filtering, and negative reinforcement logic

This commit is contained in:
2026-04-25 13:09:12 +02:00
parent ad1af4edfe
commit 77e8251aa7
43 changed files with 4107 additions and 4075 deletions

View File

@@ -0,0 +1,39 @@
from unittest.mock import patch
import pytest
from GramAddict.core.perception.action_memory import ActionMemory
from GramAddict.core.perception.spatial_parser import SpatialNode
@pytest.fixture
def memory():
with patch("GramAddict.core.qdrant_memory.UIMemoryDB") as MockDB:
mock_db = MockDB.return_value
yield ActionMemory(ui_memory=mock_db)
def test_track_click_stores_memory(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
assert memory._last_click_context is not None
assert memory._last_click_context["intent"] == "tap test"
def test_confirm_click_boosts_confidence(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
memory.confirm_click()
memory.ui_memory.boost_confidence.assert_called_once()
assert memory._last_click_context is None
def test_reject_click_decays_confidence(memory):
node = SpatialNode(bounds=(0, 0, 10, 10), content_desc="Test Node")
memory.track_click("tap test", node)
memory.reject_click()
memory.ui_memory.decay_confidence.assert_called_once()
assert memory._last_click_context is None

View File

@@ -0,0 +1,37 @@
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialNode
def test_intent_resolver_finds_bottom_tab():
resolver = IntentResolver()
# A tab at the top
top_tab = SpatialNode(bounds=(0, 0, 100, 100), content_desc="Explore Tab", clickable=True)
# A tab at the bottom
bottom_tab = SpatialNode(bounds=(0, 2200, 100, 2300), content_desc="Explore Tab", clickable=True)
# Intent resolver should prefer the one that geometrically matches the bottom navigation area
best_match = resolver.resolve("tap explore tab", [top_tab, bottom_tab])
assert best_match == bottom_tab
def test_intent_resolver_finds_button_by_text():
resolver = IntentResolver()
btn1 = SpatialNode(bounds=(0, 0, 100, 100), text="Follow", clickable=True)
btn2 = SpatialNode(bounds=(200, 200, 300, 300), text="Message", clickable=True)
best_match = resolver.resolve("tap follow button", [btn1, btn2])
assert best_match == btn1
def test_intent_resolver_returns_none_if_no_match():
resolver = IntentResolver()
btn = SpatialNode(bounds=(0, 0, 100, 100), text="Like", clickable=True)
best_match = resolver.resolve("tap follow button", [btn])
assert best_match is None

View File

@@ -0,0 +1,77 @@
from GramAddict.core.perception.spatial_parser import SpatialNode, SpatialParser
XML_FIXTURE = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" bounds="[0,0][1080,2400]" class="android.widget.FrameLayout">
<node index="0" bounds="[0,2200][1080,2400]" class="android.view.ViewGroup" content-desc="Bottom Navigation">
<node index="0" bounds="[50,2250][150,2350]" class="android.widget.ImageView" content-desc="Home Tab" clickable="true"/>
<node index="1" bounds="[250,2250][350,2350]" class="android.widget.ImageView" content-desc="Explore Tab" clickable="true"/>
</node>
<node index="1" bounds="[0,100][1080,2200]" class="androidx.recyclerview.widget.RecyclerView">
<node index="0" bounds="[50,150][1030,800]" class="android.widget.FrameLayout" content-desc="Post 1">
<node index="0" bounds="[100,700][200,750]" class="android.widget.Button" text="Like" clickable="true"/>
</node>
</node>
</node>
</hierarchy>
"""
class TestSpatialParser:
def test_parses_xml_into_spatial_nodes(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
assert root is not None
assert isinstance(root, SpatialNode)
assert root.bounds == (0, 0, 1080, 2400)
assert len(root.children) == 2 # The Bottom Nav and the Recycler View
def test_extracts_all_clickable_nodes(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
clickables = parser.get_clickable_nodes(root)
assert len(clickables) == 3
descriptions = [n.content_desc or n.text for n in clickables]
assert "Home Tab" in descriptions
assert "Explore Tab" in descriptions
assert "Like" in descriptions
def test_spatial_containment(self):
parser = SpatialParser()
root = parser.parse(XML_FIXTURE)
# Get the first post
post_nodes = [n for n in parser.get_all_nodes(root) if n.content_desc == "Post 1"]
assert len(post_nodes) == 1
post = post_nodes[0]
# Get the Like button
like_nodes = [n for n in parser.get_all_nodes(root) if n.text == "Like"]
assert len(like_nodes) == 1
like_btn = like_nodes[0]
# Spatial Containment: The Like button should be mathematically inside the Post
assert post.contains(like_btn) is True
# The Like button should NOT contain the Post
assert like_btn.contains(post) is False
# The Home Tab should NOT be in the Post
home_tabs = [n for n in parser.get_all_nodes(root) if n.content_desc == "Home Tab"]
assert post.contains(home_tabs[0]) is False
def test_spatial_intersection(self):
parser = SpatialParser()
# Node 1: Left side
n1 = SpatialNode(bounds=(0, 0, 100, 100))
# Node 2: Overlapping Right side
n2 = SpatialNode(bounds=(50, 50, 150, 150))
# Node 3: Far away
n3 = SpatialNode(bounds=(200, 200, 300, 300))
assert n1.intersects(n2) is True
assert n2.intersects(n1) is True
assert n1.intersects(n3) is False