feat(diagnostics): dump screenshots with xmls and limit retention to 5

benchmarks/run_competitive_benchmark.py

@@ -100,7 +100,7 @@ def get_installed_ollama_models():
         return []
 
 
-def benchmark_model(model_name: str, url: str, force: bool = False):
+def benchmark_model(model_name: str, url: str, force: bool = False, iterations: int = 3):
     db = load_json(BENCHMARKS_FILE) or {"models": {}}
     scenarios_data = load_json(SCENARIOS_FILE)
     if not scenarios_data:
@@ -138,49 +138,69 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
             "Return: {\"index\": number, \"reason\": \"...\"}"
         )
 
-        start_time = time.time()
-        try:
-            resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
-            latency = int((time.time() - start_time) * 1000)
-            total_latency += latency
-        except Exception as e:
-            print(f"  ❌ API Request failed for scenario {scenario['id']}: {e}")
-            passed_all = False
-            continue
+        scenario_latencies = []
+        scenario_scores = []
+        successes = 0
 
-        raw_points = 0
-        try:
-            clean = resp_str.strip()
-            if clean.startswith("```json"):
-                clean = clean[7:]
-            if clean.endswith("```"):
-                clean = clean[:-3]
-            data = json.loads(clean)
-
-            # Points for structural adherence
-            if "index" in data and "reason" in data:
-                raw_points += 40
-
-                # Points for correctness
-                if data["index"] == scenario["target_index"]:
-                    raw_points += 60
-                    print(f"  ✅ Correct index ({data['index']}).")
-                else:
-                    passed_all = False
-                    print(f"  ❌ Wrong index ({data['index']}). Target was {scenario['target_index']}.")
-            else:
+        for _ in range(iterations):
+            start_time = time.time()
+            try:
+                resp_str = query_telepathic_llm(model_name, url, system_prompt, user_prompt)
+                latency = int((time.time() - start_time) * 1000)
+                scenario_latencies.append(latency)
+            except Exception as e:
+                print(f"  ❌ API Request failed for scenario {scenario['id']}: {e}")
                 passed_all = False
-                print("  ❌ JSON missing fields.")
-        except Exception:
-            passed_all = False
-            print("  ❌ JSON Parsing failed.")
+                continue
 
-        results_detail[scenario["id"]] = raw_points
-        total_raw += raw_points
+            raw_points = 0
+            try:
+                clean = resp_str.strip()
+                if clean.startswith("```json"):
+                    clean = clean[7:]
+                if clean.endswith("```"):
+                    clean = clean[:-3]
+                data = json.loads(clean)
+
+                # Points for structural adherence
+                if "index" in data and "reason" in data:
+                    raw_points += 40
+
+                    # Points for correctness
+                    if data["index"] == scenario["target_index"]:
+                        raw_points += 60
+                        successes += 1
+                    else:
+                        print(f"  ❌ Wrong index ({data.get('index')}). Target was {scenario['target_index']}.")
+                else:
+                    print("  ❌ JSON missing fields.")
+            except Exception:
+                print("  ❌ JSON Parsing failed.")
+
+            scenario_scores.append(raw_points)
+
+        avg_scenario_score = int(sum(scenario_scores) / len(scenario_scores)) if scenario_scores else 0
+        avg_scenario_latency = int(sum(scenario_latencies) / len(scenario_latencies)) if scenario_latencies else 0
+
+        pass_rate = (successes / iterations) * 100
+        if pass_rate < 100.0:
+            passed_all = False
+
+        print(
+            f"  Result: {pass_rate:.0f}% Pass Rate | Avg Score: {avg_scenario_score}/100 | Avg Latency: {avg_scenario_latency}ms"
+        )
+
+        results_detail[scenario["id"]] = {
+            "avg_score": avg_scenario_score,
+            "pass_rate": pass_rate,
+            "latency": avg_scenario_latency,
+        }
+        total_raw += avg_scenario_score
+        total_latency += avg_scenario_latency
 
     avg_latency = total_latency // len(scenarios) if scenarios else 0
     print(
-        f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Score: {total_raw} | Latency: {avg_latency}ms"
+        f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Avg Score: {total_raw} | Latency: {avg_latency}ms"
     )
 
     if model_name not in db["models"]:
@@ -212,6 +232,9 @@ if __name__ == "__main__":
     parser.add_argument("--url", type=str, help="Explicit endpoint URL")
     parser.add_argument("--force", action="store_true", help="Force re-testing")
     parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
+    parser.add_argument(
+        "--iterations", type=int, default=3, help="Number of iterations per scenario to measure reliability"
+    )
 
     args, unknown = parser.parse_known_args()
 
@@ -241,5 +264,5 @@ if __name__ == "__main__":
         sys.exit(1)
 
     for m, u in set(models_to_test):
-        benchmark_model(m, u, args.force)
+        benchmark_model(m, u, args.force, args.iterations)
         time.sleep(1)