update
This commit is contained in:
196
benchmarks/ai_memory_diagnostics.py
Normal file
196
benchmarks/ai_memory_diagnostics.py
Normal file
@@ -0,0 +1,196 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
|
||||
# Root path alignment
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(ROOT_DIR)
|
||||
|
||||
def colored(text, color, attrs=None):
|
||||
colors = {
|
||||
"red": "\033[91m", "green": "\033[92m", "yellow": "\033[93m",
|
||||
"blue": "\033[94m", "magenta": "\033[95m", "cyan": "\033[96m",
|
||||
"white": "\033[97m"
|
||||
}
|
||||
reset = "\033[0m"
|
||||
bold = "\033[1m" if attrs and "bold" in attrs else ""
|
||||
return f"{bold}{colors.get(color, '')}{text}{reset}"
|
||||
from GramAddict.core.config import Config
|
||||
from GramAddict.core.qdrant_memory import ParasocialCRMDB, CommentMemoryDB
|
||||
from GramAddict.core.resonance_engine import ResonanceEngine
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
class MockArgs:
|
||||
def __init__(self):
|
||||
self.ai_vibe = "friendly, authentic, travel, photography"
|
||||
self.ai_blacklist_topics = "onlyfans, bitcoin, crypto, nsfw, spam, give-away"
|
||||
self.ai_learn_comments = True
|
||||
self.ai_condenser_model = "qwen3.5:latest"
|
||||
self.ai_condenser_url = "http://localhost:11434/api/generate"
|
||||
self.ai_vision_navigation = False
|
||||
self.ai_vision_context = False
|
||||
|
||||
class MockConfig:
|
||||
def __init__(self):
|
||||
self.args = MockArgs()
|
||||
|
||||
class AIMemoryDiagnosticRunner:
|
||||
def __init__(self):
|
||||
self.configs = MockConfig()
|
||||
Config().args = self.configs.args
|
||||
self.crm_db = ParasocialCRMDB()
|
||||
self.comment_db = CommentMemoryDB()
|
||||
self.resonance_oracle = ResonanceEngine("benchmark_agent", crm=self.crm_db)
|
||||
|
||||
def setup(self):
|
||||
print(colored("🧹 Initializing benchmark data...", "cyan"))
|
||||
# We handle unique targets so we don't wipe the DB
|
||||
pass
|
||||
|
||||
def test_rag_comment_extraction(self) -> dict:
|
||||
"""
|
||||
Challenge: Pass an XML dump with real comments and toxic OnlyFans/Crypto spam.
|
||||
Verify that the LLM Condenser drops the junk and stores the good comments.
|
||||
"""
|
||||
fixture_path = os.path.join(ROOT_DIR, "tests", "fixtures", "comments_mock.xml")
|
||||
with open(fixture_path, "r", encoding="utf-8") as f:
|
||||
xml_data = f.read()
|
||||
|
||||
print(colored(f" -> Extracing comments using RAG Condenser ({self.configs.args.ai_condenser_model})...", "yellow"))
|
||||
start = time.time()
|
||||
|
||||
# Intercept the database write to bypass Qdrant indexing limits and solely test RAG filter logic
|
||||
intercepted_comments = []
|
||||
|
||||
def mock_log(self, text: str, vibe: str, author: str = "unknown"):
|
||||
intercepted_comments.append(text)
|
||||
|
||||
try:
|
||||
from unittest.mock import patch
|
||||
with patch.object(CommentMemoryDB, 'store_comment', new=mock_log):
|
||||
# Override the author logic
|
||||
test_author = f"benchmark_source_{int(time.time())}"
|
||||
self.resonance_oracle.extract_and_learn_comments(xml_data, self.configs, author=test_author)
|
||||
time.sleep(1.0)
|
||||
except Exception as e:
|
||||
print(f"❌ EXCEPTION: {e}")
|
||||
return {"passed": False, "reason": str(e)}
|
||||
|
||||
try:
|
||||
learned_texts = [c.lower() for c in intercepted_comments]
|
||||
dur = time.time() - start
|
||||
print(colored(f" -> Intercepted: {learned_texts}", "yellow"))
|
||||
|
||||
toxic_count = sum(1 for t in learned_texts if "onlyfans" in t or "bitcoin" in t or "dm" in t or "$" in t)
|
||||
good_count = sum(1 for t in learned_texts if "majestic" in t or "lighting" in t)
|
||||
|
||||
if toxic_count > 0:
|
||||
print(colored(" ❌ [Sub-Test] LLM Condenser hallucinated or failed to block toxic queries (OnlyFans/Crypto).", "red"))
|
||||
return {"passed": False, "reason": "Toxic comments leaked"}
|
||||
|
||||
if good_count == 0:
|
||||
print(colored(" ❌ [Sub-Test] LLM Condenser stripped everything or crashed. No good comments persisted.", "red"))
|
||||
return {"passed": False, "reason": "Good comments dropped"}
|
||||
|
||||
print(colored(f" ✅ [Sub-Test] RAG Filter passed! 0 toxic comments, {good_count} valid comments mapped. Latency {dur:.2f}s", "green"))
|
||||
return {"passed": True, "reason": "Toxic filtered, good preserved."}
|
||||
|
||||
except Exception as e:
|
||||
return {"passed": False, "reason": f"DB Error: {e}"}
|
||||
|
||||
def test_crm_profile_context(self) -> dict:
|
||||
"""
|
||||
Challenge: Parse and persist profile data into the CRM safely.
|
||||
"""
|
||||
target = "benchmark_target"
|
||||
context_string = "234 Posts | 1.2M Followers | 🏔️ Alpine Photographer | Link in bio"
|
||||
|
||||
try:
|
||||
self.crm_db.log_profile_context(target, context_string)
|
||||
time.sleep(0.5) # indexing buffer
|
||||
|
||||
history = self.crm_db.get_conversation_context(target)
|
||||
if context_string in history or "1.2M Followers" in history:
|
||||
print(colored(" ✅ [Sub-Test] Profile context cleanly injected into RAG CRM payload.", "green"))
|
||||
return {"passed": True, "reason": "Context string found."}
|
||||
else:
|
||||
return {"passed": False, "reason": "Profile context missing from CRM retrieval."}
|
||||
|
||||
except Exception as e:
|
||||
return {"passed": False, "reason": str(e)}
|
||||
|
||||
def test_crm_interaction_evolution(self) -> dict:
|
||||
"""
|
||||
Challenge: Push 3 sequential interactions for a user to see if the CRM stage evolves (0 -> 1 -> 2 -> 3).
|
||||
"""
|
||||
target = "benchmark_target"
|
||||
try:
|
||||
print(" -> Interacting: 'Like'")
|
||||
self.crm_db.log_interaction(target, "tap_like_button", new_stage=1)
|
||||
time.sleep(0.1)
|
||||
print(" -> Interacting: 'Follow'")
|
||||
self.crm_db.log_interaction(target, "tap_follow_button", new_stage=2)
|
||||
time.sleep(0.1)
|
||||
print(" -> Interacting: 'Comment'")
|
||||
self.crm_db.log_generated_comment(target, "Wow great photo!")
|
||||
self.crm_db.log_interaction(target, "tap_comment_button", new_stage=3)
|
||||
time.sleep(0.5)
|
||||
|
||||
stage_info = self.crm_db.get_relationship_stage(target)
|
||||
stage = stage_info.get("stage", 0)
|
||||
|
||||
if stage >= 3:
|
||||
print(colored(f" ✅ [Sub-Test] CRM safely advanced state memory to Stage {stage}.", "green"))
|
||||
return {"passed": True, "reason": "Evolution logic passed."}
|
||||
else:
|
||||
print(colored(f" ❌ [Sub-Test] CRM stalled at Stage {stage}!", "red"))
|
||||
return {"passed": False, "reason": "Failed to evolve stage"}
|
||||
|
||||
except Exception as e:
|
||||
return {"passed": False, "reason": str(e)}
|
||||
|
||||
def execute_all(self):
|
||||
self.setup()
|
||||
results = {
|
||||
"timestamp": time.time(),
|
||||
"model": self.configs.args.ai_condenser_model,
|
||||
"scenarios": {}
|
||||
}
|
||||
|
||||
def run_and_log(name, func):
|
||||
print(colored(f"\n--- SCENARIO: {name} ---", "magenta"))
|
||||
start_time = time.time()
|
||||
data = {"passed": False, "reason": "Unknown error", "latency_ms": 0}
|
||||
try:
|
||||
res = func()
|
||||
if isinstance(res, dict): data.update(res)
|
||||
elif res is True: data["passed"] = True
|
||||
except Exception as e:
|
||||
print(colored(f"❌ EXCEPTION: {e}", "red"))
|
||||
data["reason"] = str(e)
|
||||
|
||||
dur = time.time() - start_time
|
||||
data["latency_ms"] = int(dur * 1000)
|
||||
results["scenarios"][name] = data
|
||||
|
||||
if data["passed"]:
|
||||
print(colored(f"🏁 {name} completed successfully in {dur:.2f}s", "green"))
|
||||
else:
|
||||
print(colored(f"🚨 {name} FAILED! (Elapsed: {dur:.2f}s)", "red", attrs=["bold"]))
|
||||
print(colored(f" Reason: {data['reason']}", "yellow"))
|
||||
|
||||
run_and_log("RAG Comment Blacklist Extraction", self.test_rag_comment_extraction)
|
||||
run_and_log("CRM Profile Context Injection", self.test_crm_profile_context)
|
||||
run_and_log("CRM Sequential Evolution", self.test_crm_interaction_evolution)
|
||||
|
||||
self.setup() # Teardown
|
||||
|
||||
out_path = os.path.join(ROOT_DIR, "benchmarks", "data", "ai_memory_results.json")
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(results, f, indent=4)
|
||||
print(colored(f"\n📄 Saved AI Memory Benchmark results to: {out_path}", "cyan", attrs=["bold"]))
|
||||
|
||||
if __name__ == "__main__":
|
||||
runner = AIMemoryDiagnosticRunner()
|
||||
runner.execute_all()
|
||||
231
benchmarks/live_brain_diagnostics.py
Normal file
231
benchmarks/live_brain_diagnostics.py
Normal file
@@ -0,0 +1,231 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from colorama import Fore, Style, init
|
||||
|
||||
# Init Colorama for cross-platform color support
|
||||
init(autoreset=True)
|
||||
|
||||
# Ensure root is in path
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT_DIR)
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
from GramAddict.core.qdrant_memory import UIMemoryDB
|
||||
|
||||
# Mute noisy loggers
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def colored(text, color, attrs=None):
|
||||
c = getattr(Fore, color.upper(), "")
|
||||
attr_str = ""
|
||||
if attrs and "bold" in attrs:
|
||||
attr_str = Style.BRIGHT
|
||||
return f"{attr_str}{c}{text}"
|
||||
|
||||
class MockArgs:
|
||||
def __init__(self):
|
||||
self.ai_telepathic_model = "qwen3.5:latest"
|
||||
self.ai_telepathic_url = "http://localhost:11434/api/generate"
|
||||
self.ai_embedding_model = "nomic-embed-text"
|
||||
self.ai_embedding_url = "http://localhost:11434/api/embeddings"
|
||||
self.ai_vision_navigation = True
|
||||
self.ai_vision_context = True
|
||||
|
||||
import base64
|
||||
class MockDevice:
|
||||
def __init__(self):
|
||||
self.args = MockArgs()
|
||||
self.app_id = "com.instagram.android"
|
||||
|
||||
def screenshot(self):
|
||||
# Return a simple 1x1 black pixel PNG to test the True Vision payload mapping
|
||||
# without crashing on invalid image data.
|
||||
return base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAXSURBVBhXY3jP4PgfAAWEAziO3O8MAAAAASUVORK5CYII=")
|
||||
|
||||
from GramAddict.core.config import Config
|
||||
Config().args = MockArgs()
|
||||
|
||||
class BrainDiagnosticRunner:
|
||||
"""
|
||||
Professional diagnostic suite for Live integration testing of the
|
||||
Singularity LLM Cognitive Stack and Vector DB (Qdrant) persistence.
|
||||
Tested against heavy real-world XML dumps from Instagram.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.device = MockDevice()
|
||||
self.engine = TelepathicEngine.get_instance()
|
||||
self.mem_db = UIMemoryDB()
|
||||
|
||||
# Test Namespaces
|
||||
self.intents = {
|
||||
"modal": "diagnostics_dismiss_obstacle",
|
||||
"ad": "diagnostics_find_sponsored",
|
||||
"hallucination": "diagnostics_tap_like_button",
|
||||
"unfollow": "diagnostics_tap_following_button"
|
||||
}
|
||||
|
||||
# Load heavy real-world XML files
|
||||
self.fixtures_dir = os.path.join(ROOT_DIR, "tests", "fixtures")
|
||||
self.xmls = {
|
||||
"modal": self._load_fixture("blocked_ui.xml"),
|
||||
"ad": self._load_fixture("peugeot_ad.xml"),
|
||||
"hallucination": self._load_fixture("vlm_hallucination.xml"),
|
||||
"unfollow": self._load_fixture("unfollow_list_dump.xml")
|
||||
}
|
||||
|
||||
def _load_fixture(self, filename) -> str:
|
||||
path = os.path.join(self.fixtures_dir, filename)
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(f"Fixture {filename} completely missing. Cannot run parcours.")
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
def setup(self):
|
||||
print(colored("🧠 STARTING LIVE BRAIN 'PARCOURS' DIAGNOSTICS (Qdrant + Qwen 3.5)", "cyan", attrs=["bold"]))
|
||||
if not self.mem_db.is_connected:
|
||||
logger.error("❌ Qdrant is offline! Diagnostics cannot proceed.")
|
||||
sys.exit(1)
|
||||
|
||||
print(colored("🧹 Initializing diagnostic namespace (clearing old cache)...", "yellow"))
|
||||
for intent in self.intents.values():
|
||||
pt_id = self.mem_db._deterministic_id(intent)
|
||||
self.mem_db._delete_point(pt_id)
|
||||
|
||||
def teardown(self):
|
||||
print(colored("🧹 Tearing down diagnostic namespace...", "yellow"))
|
||||
for intent in self.intents.values():
|
||||
pt_id = self.mem_db._deterministic_id(intent)
|
||||
self.mem_db._delete_point(pt_id)
|
||||
print(colored("✅ Diagnostics Complete.", "green", attrs=["bold"]))
|
||||
|
||||
def test_modal_trap(self) -> dict:
|
||||
"""
|
||||
Challenge: Find the 'OK' or 'Dismiss' button on a blocking popup.
|
||||
"""
|
||||
xml = self.xmls["modal"]
|
||||
intent = self.intents["modal"]
|
||||
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
|
||||
|
||||
if not node:
|
||||
print(colored(" ❌ LLM failed to find the dismiss button entirely.", "red"))
|
||||
return {"passed": False, "reason": "No node found"}
|
||||
|
||||
semantic = str(node.get("semantic", "")).lower()
|
||||
if "try again later" in semantic or "action block" in semantic:
|
||||
print(colored(" ❌ LLM selected the title text instead of the dismiss button.", "red"))
|
||||
return {"passed": False, "reason": "Selected title instead of button"}
|
||||
|
||||
if "dismiss" in semantic or "ok" in semantic:
|
||||
print(colored(f" ✅ VLM correctly reasoned the popup OK/Dismiss button: {semantic}", "green"))
|
||||
return {"passed": True, "reason": f"Found correct button: {semantic}"}
|
||||
|
||||
return {"passed": False, "reason": f"Selected unrelated element: {semantic}"}
|
||||
|
||||
def test_ad_deception(self) -> dict:
|
||||
"""
|
||||
Challenge: Identify if the post is an ad by finding 'Sponsored' text.
|
||||
"""
|
||||
xml = self.xmls["ad"]
|
||||
intent = self.intents["ad"]
|
||||
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
|
||||
|
||||
if not node:
|
||||
print(colored(" ❌ LLM failed to identify the sponsored indicator.", "red"))
|
||||
return {"passed": False, "reason": "Missed sponsored text"}
|
||||
|
||||
semantic = str(node.get("semantic", "")).lower()
|
||||
if "sponsored" in semantic:
|
||||
print(colored(" ✅ VLM correctly identified the tiny 'Sponsored' label amidst a huge post.", "green"))
|
||||
|
||||
# --- Test Fast Path Recall Sub-Scenario ---
|
||||
# Save it
|
||||
self.engine.confirm_click(intent)
|
||||
self.mem_db.store_memory(intent, xml, node)
|
||||
import time
|
||||
time.sleep(0.5)
|
||||
# Try to grab it again
|
||||
start = time.time()
|
||||
recall_node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
|
||||
dur = time.time() - start
|
||||
if recall_node and recall_node.get("source") == "memory":
|
||||
print(colored(f" ✅ [Sub-Test] Instant Qdrant Memory Recall verified! Latency: {dur:.3f}s", "green"))
|
||||
return {"passed": True, "reason": "Identified sponsored text and verified memory loop."}
|
||||
else:
|
||||
print(colored(" ❌ [Sub-Test] Memory recall failed.", "red"))
|
||||
return {"passed": False, "reason": "Found ad, but memory persistence failed."}
|
||||
|
||||
return {"passed": False, "reason": f"Picked wrong node: {semantic}"}
|
||||
|
||||
def test_vlm_hallucination(self) -> dict:
|
||||
"""
|
||||
Challenge: Find the like heart icon, ignoring caption text that says 'LIKE'.
|
||||
"""
|
||||
xml = self.xmls["hallucination"]
|
||||
intent = self.intents["hallucination"]
|
||||
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
|
||||
|
||||
if not node:
|
||||
print(colored(" ❌ LLM failed to find any like button.", "red"))
|
||||
return {"passed": False, "reason": "No node found"}
|
||||
|
||||
semantic = str(node.get("semantic", "")).lower()
|
||||
|
||||
is_caption = ("double tap" in semantic or "like" in semantic) and "row feed button" not in semantic
|
||||
if is_caption:
|
||||
print(colored(" ❌ LLM fell for the semantic hallucination gap and selected the text caption!", "red"))
|
||||
return {"passed": False, "reason": "Fell for caption text trap"}
|
||||
|
||||
if "row feed button like" in semantic or "heart" in semantic:
|
||||
print(colored(" ✅ VLM successfully ignored the deceptive caption and found the structural like button.", "green"))
|
||||
return {"passed": True, "reason": "Ignored text trap, clicked structural button"}
|
||||
|
||||
return {"passed": False, "reason": f"Picked unrelated node: {semantic}"}
|
||||
|
||||
def execute_all(self):
|
||||
self.setup()
|
||||
results = {
|
||||
"timestamp": time.time(),
|
||||
"model": self.device.args.ai_telepathic_model,
|
||||
"scenarios": {}
|
||||
}
|
||||
|
||||
def run_and_log(name, func):
|
||||
print(colored(f"\n--- SCENARIO: {name} ---", "magenta"))
|
||||
start_time = time.time()
|
||||
data = {"passed": False, "reason": "Unknown error", "latency_ms": 0}
|
||||
try:
|
||||
res = func()
|
||||
if isinstance(res, dict): data.update(res)
|
||||
elif res is True: data["passed"] = True
|
||||
except Exception as e:
|
||||
print(colored(f"❌ EXCEPTION: {e}", "red"))
|
||||
data["reason"] = str(e)
|
||||
|
||||
dur = time.time() - start_time
|
||||
data["latency_ms"] = int(dur * 1000)
|
||||
results["scenarios"][name] = data
|
||||
|
||||
if data["passed"]:
|
||||
print(colored(f"🏁 {name} completed successfully in {dur:.2f}s", "green"))
|
||||
else:
|
||||
print(colored(f"🚨 {name} FAILED! (Elapsed: {dur:.2f}s)", "red", attrs=["bold"]))
|
||||
|
||||
run_and_log("The Modal Trap (Blocked UI)", self.test_modal_trap)
|
||||
run_and_log("The Ad Deception (Sponsored)", self.test_ad_deception)
|
||||
run_and_log("The VLM Hallucination Gap (Text Trap)", self.test_vlm_hallucination)
|
||||
self.teardown()
|
||||
|
||||
out_path = os.path.join(ROOT_DIR, "benchmarks", "data", "live_learning_results.json")
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(results, f, indent=4)
|
||||
print(colored(f"\n📄 Saved intensive learning results to: {out_path}", "cyan", attrs=["bold"]))
|
||||
|
||||
if __name__ == "__main__":
|
||||
runner = BrainDiagnosticRunner()
|
||||
runner.execute_all()
|
||||
230
benchmarks/run_competitive_benchmark.py
Normal file
230
benchmarks/run_competitive_benchmark.py
Normal file
@@ -0,0 +1,230 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
|
||||
# Add root project path so we can import internal modules safely
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
BENCHMARKS_FILE = os.path.join(os.path.dirname(__file__), "data/llm_benchmarks.json")
|
||||
SCENARIOS_FILE = os.path.join(os.path.dirname(__file__), "data/benchmark_scenarios.json")
|
||||
|
||||
def load_json(path):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def save_json(path, data):
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
||||
def normalize_scores(db):
|
||||
if not db.get("models"):
|
||||
return db
|
||||
|
||||
# 1. Find the highest raw score across all models
|
||||
max_raw = 0
|
||||
leader_model = None
|
||||
|
||||
for name, data in db["models"].items():
|
||||
if data.get("is_unsuitable"):
|
||||
continue
|
||||
|
||||
raw = data.get("raw_score", 0)
|
||||
if raw > max_raw:
|
||||
max_raw = raw
|
||||
leader_model = name
|
||||
elif raw == max_raw and max_raw > 0:
|
||||
# Tie-breaker: Latency
|
||||
current_lat = data.get("latency_ms", 99999)
|
||||
leader_lat = db["models"][leader_model].get("latency_ms", 99999)
|
||||
if current_lat < leader_lat:
|
||||
leader_model = name
|
||||
|
||||
if max_raw == 0:
|
||||
return db
|
||||
|
||||
# 2. Update relative performance
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
|
||||
data["is_leader"] = (name == leader_model)
|
||||
|
||||
return db
|
||||
|
||||
def get_installed_ollama_models():
|
||||
"""
|
||||
Finds truly local Ollama models by parsing 'ollama list'.
|
||||
Strictly excludes remote/cloud endpoints or embedding-only models.
|
||||
"""
|
||||
try:
|
||||
output = subprocess.check_output(["/usr/local/bin/ollama", "list"]).decode("utf-8")
|
||||
models = []
|
||||
for line in output.split("\n")[1:]:
|
||||
if line.strip():
|
||||
# Format: NAME, ID, SIZE, MODIFIED
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
name = parts[0]
|
||||
size = parts[2]
|
||||
|
||||
# 1. Skip if size is '-' (remote/cloud model)
|
||||
if size == "-":
|
||||
continue
|
||||
|
||||
# 2. Skip ':cloud' tagged models explicitly
|
||||
if ":cloud" in name:
|
||||
continue
|
||||
|
||||
# 3. Filter out purely embedding models
|
||||
if any(k in name.lower() for k in ["embed", "minilm", "rerank"]):
|
||||
continue
|
||||
|
||||
models.append(name)
|
||||
return models
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not list Ollama models: {e}")
|
||||
return []
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False):
|
||||
db = load_json(BENCHMARKS_FILE) or {"models": {}}
|
||||
scenarios_data = load_json(SCENARIOS_FILE)
|
||||
if not scenarios_data:
|
||||
print("❌ Scenarios file missing!")
|
||||
return
|
||||
|
||||
if not force and model_name in db.get("models", {}):
|
||||
pct = db["models"][model_name].get("relative_performance_pct", "N/A")
|
||||
if not db["models"][model_name].get("is_unsuitable"):
|
||||
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
|
||||
return
|
||||
|
||||
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name}")
|
||||
|
||||
total_raw = 0
|
||||
total_latency = 0
|
||||
results_detail = {}
|
||||
passed_all = True
|
||||
|
||||
blank_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
|
||||
"Output ONLY valid JSON: {\"index\": number, \"reason\": \"brief reason\"}"
|
||||
)
|
||||
|
||||
scenarios = scenarios_data["scenarios"]
|
||||
for scenario in scenarios:
|
||||
print(f"--- Running: {scenario['name']} ---")
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
"Return: {\"index\": number, \"reason\": \"...\"}"
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
total_latency += latency
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed for scenario {scenario['id']}: {e}")
|
||||
passed_all = False
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"): clean = clean[7:]
|
||||
if clean.endswith("```"): clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
# Points for structural adherence
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
print(f" ✅ Correct index ({data['index']}).")
|
||||
else:
|
||||
passed_all = False
|
||||
print(f" ❌ Wrong index ({data['index']}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
passed_all = False
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
passed_all = False
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
results_detail[scenario["id"]] = raw_points
|
||||
total_raw += raw_points
|
||||
|
||||
avg_latency = total_latency // len(scenarios) if scenarios else 0
|
||||
print(f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Score: {total_raw} | Latency: {avg_latency}ms")
|
||||
|
||||
if model_name not in db["models"]:
|
||||
db["models"][model_name] = {}
|
||||
|
||||
db["models"][model_name].update({
|
||||
"raw_score": total_raw,
|
||||
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
|
||||
"latency_ms": avg_latency,
|
||||
"last_tested": datetime.utcnow().isoformat() + "Z",
|
||||
"details": results_detail,
|
||||
"passed_all": passed_all,
|
||||
"is_unsuitable": not passed_all
|
||||
})
|
||||
|
||||
# Recalculate relative scores across all models
|
||||
db = normalize_scores(db)
|
||||
save_json(BENCHMARKS_FILE, db)
|
||||
|
||||
if __name__ == "__main__":
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
parser = argparse.ArgumentParser(description="Competitive Benchmark for Singularity", add_help=False)
|
||||
parser.add_argument("--config", type=str, help="Bot config file")
|
||||
parser.add_argument("--model", type=str, help="Explicit model name")
|
||||
parser.add_argument("--url", type=str, help="Explicit endpoint URL")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-testing")
|
||||
parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
models_to_test = []
|
||||
|
||||
if args.all_ollama:
|
||||
ollama_models = get_installed_ollama_models()
|
||||
for m in ollama_models:
|
||||
models_to_test.append((m, "http://localhost:11434/api/generate"))
|
||||
elif args.model and args.url:
|
||||
models_to_test.append((args.model, args.url))
|
||||
elif args.config:
|
||||
configs = Config(first_run=True, config=args.config)
|
||||
configs.parse_args()
|
||||
|
||||
for attr, pref in [("ai_telepathic_model", "ai_telepathic_url"), ("ai_model", "ai_model_url"), ("ai_condenser_model", "ai_condenser_url")]:
|
||||
m = getattr(configs.args, attr, None)
|
||||
u = getattr(configs.args, pref, "http://localhost:11434/api/generate")
|
||||
if m:
|
||||
models_to_test.append((m, u))
|
||||
else:
|
||||
print("❌ Syntax: --all-ollama OR --config test_config.yml OR --model x --url y")
|
||||
sys.exit(1)
|
||||
|
||||
for m, u in set(models_to_test):
|
||||
benchmark_model(m, u, args.force)
|
||||
time.sleep(1)
|
||||
Reference in New Issue
Block a user