update
This commit is contained in:
@@ -1,182 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
# Add root project path so we can import internal modules safely
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
|
||||
BENCHMARKS_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)), "GramAddict/core/llm_benchmarks.json")
|
||||
SCENARIOS_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)), "GramAddict/core/benchmark_scenarios.json")
|
||||
|
||||
def load_json(path):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def save_json(path, data):
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
||||
def normalize_scores(db):
|
||||
if not db.get("models"):
|
||||
return db
|
||||
|
||||
# 1. Find the highest raw score across all models
|
||||
max_raw = 0
|
||||
leader_model = None
|
||||
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
if raw > max_raw:
|
||||
max_raw = raw
|
||||
leader_model = name
|
||||
elif raw == max_raw and max_raw > 0:
|
||||
# Tie-breaker: Latency
|
||||
current_lat = data.get("latency_ms", 99999)
|
||||
leader_lat = db["models"][leader_model].get("latency_ms", 99999)
|
||||
if current_lat < leader_lat:
|
||||
leader_model = name
|
||||
|
||||
if max_raw == 0:
|
||||
return db
|
||||
|
||||
# 2. Update relative performance
|
||||
for name, data in db["models"].items():
|
||||
raw = data.get("raw_score", 0)
|
||||
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
|
||||
data["is_leader"] = (name == leader_model)
|
||||
|
||||
return db
|
||||
|
||||
def benchmark_model(model_name: str, url: str, force: bool = False):
|
||||
db = load_json(BENCHMARKS_FILE) or {"models": {}}
|
||||
scenarios_data = load_json(SCENARIOS_FILE)
|
||||
if not scenarios_data:
|
||||
print("❌ Scenarios file missing!")
|
||||
return
|
||||
|
||||
if not force and model_name in db.get("models", {}):
|
||||
pct = db["models"][model_name].get("relative_performance_pct", "N/A")
|
||||
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
|
||||
return
|
||||
|
||||
print(f"🚀 [Competitive Benchmarking] Model: {model_name}")
|
||||
|
||||
total_raw = 0
|
||||
total_latency = 0
|
||||
results_detail = {}
|
||||
|
||||
blank_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
|
||||
system_prompt = (
|
||||
"You identify which UI element to tap on an Android screen. "
|
||||
"Output ONLY valid JSON: {\"index\": number, \"reason\": \"brief reason\"}"
|
||||
)
|
||||
|
||||
for scenario in scenarios_data["scenarios"]:
|
||||
print(f"--- Running: {scenario['name']} ---")
|
||||
|
||||
user_prompt = (
|
||||
f"Which element should I tap to: {scenario['task']}\n\n"
|
||||
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
|
||||
"Rules:\n"
|
||||
"- Pick the SMALLEST, most specific button or icon\n"
|
||||
"- NEVER pick large containers\n"
|
||||
"Return: {\"index\": number, \"reason\": \"...\"}"
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
|
||||
latency = int((time.time() - start_time) * 1000)
|
||||
total_latency += latency
|
||||
except Exception as e:
|
||||
print(f" ❌ API Request failed for scenario {scenario['id']}: {e}")
|
||||
continue
|
||||
|
||||
raw_points = 0
|
||||
try:
|
||||
clean = resp_str.strip()
|
||||
if clean.startswith("```json"): clean = clean[7:]
|
||||
if clean.endswith("```"): clean = clean[:-3]
|
||||
data = json.loads(clean)
|
||||
|
||||
# Points for structural adherence
|
||||
if "index" in data and "reason" in data:
|
||||
raw_points += 40
|
||||
|
||||
# Points for correctness
|
||||
if data["index"] == scenario["target_index"]:
|
||||
raw_points += 60
|
||||
print(f" ✅ Correct index ({data['index']}).")
|
||||
else:
|
||||
print(f" ❌ Wrong index ({data['index']}). Target was {scenario['target_index']}.")
|
||||
else:
|
||||
print(" ❌ JSON missing fields.")
|
||||
except Exception:
|
||||
print(" ❌ JSON Parsing failed.")
|
||||
|
||||
results_detail[scenario["id"]] = raw_points
|
||||
total_raw += raw_points
|
||||
|
||||
print(f"\n📊 Total Raw Score for {model_name}: {total_raw}")
|
||||
|
||||
if model_name not in db["models"]:
|
||||
db["models"][model_name] = {}
|
||||
|
||||
db["models"][model_name].update({
|
||||
"raw_score": total_raw,
|
||||
"latency_ms": total_latency // len(scenarios_data["scenarios"]),
|
||||
"last_tested": datetime.utcnow().isoformat() + "Z",
|
||||
"details": results_detail
|
||||
})
|
||||
|
||||
# Recalculate relative scores across all models
|
||||
db = normalize_scores(db)
|
||||
save_json(BENCHMARKS_FILE, db)
|
||||
|
||||
leader_name = [n for n, d in db["models"].items() if d.get("is_leader")][0]
|
||||
rel_pct = db["models"][model_name]["relative_performance_pct"]
|
||||
|
||||
print(f"🏆 Current Leader: {leader_name}")
|
||||
print(f"✨ Relative Performance for {model_name}: {rel_pct}%")
|
||||
|
||||
if __name__ == "__main__":
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
parser = argparse.ArgumentParser(description="Competitive Benchmark for Singularity", add_help=False)
|
||||
parser.add_argument("--config", type=str, help="Bot config file")
|
||||
parser.add_argument("--model", type=str, help="Explicit model name")
|
||||
parser.add_argument("--url", type=str, help="Explicit endpoint URL")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-testing")
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
models_to_test = []
|
||||
|
||||
if args.model and args.url:
|
||||
models_to_test.append((args.model, args.url))
|
||||
elif args.config:
|
||||
configs = Config(first_run=True, config=args.config)
|
||||
configs.parse_args()
|
||||
|
||||
for attr, pref in [("ai_telepathic_model", "ai_telepathic_url"), ("ai_model", "ai_model_url"), ("ai_condenser_model", "ai_condenser_url")]:
|
||||
m = getattr(configs.args, attr, None)
|
||||
u = getattr(configs.args, pref, "https://openrouter.ai/api/v1/chat/completions")
|
||||
if m:
|
||||
models_to_test.append((m, u))
|
||||
else:
|
||||
print("❌ Syntax: --config test_config.yml or --model x --url y")
|
||||
sys.exit(1)
|
||||
|
||||
for m, u in set(models_to_test):
|
||||
benchmark_model(m, u, args.force)
|
||||
time.sleep(1)
|
||||
Reference in New Issue
Block a user