"""
Benchmark Integrity Tests
==========================

These tests ensure the benchmark infrastructure produces RELIABLE,
COMPARABLE results across model evaluations.

Covers:
1. Scenario data consistency (no mixed formats)
2. Brain-type scenarios exist and are tested via format_json=False
3. Scoring normalization (per-scenario, not raw totals)
4. Minimum iteration count enforcement
"""

import json
import os

import pytest

BENCHMARKS_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "benchmarks", "data")
SCENARIOS_FILE = os.path.join(BENCHMARKS_DIR, "benchmark_scenarios.json")
RESULTS_FILE = os.path.join(BENCHMARKS_DIR, "llm_benchmarks.json")


class TestBenchmarkScenarioIntegrity:
    """Contract: Benchmark scenarios must cover BOTH bot capabilities."""

    def test_scenarios_file_exists(self):
        assert os.path.exists(SCENARIOS_FILE), "benchmark_scenarios.json is missing!"

    def test_scenarios_have_required_fields(self):
        with open(SCENARIOS_FILE) as f:
            data = json.load(f)

        for scenario in data["scenarios"]:
            assert "id" in scenario, f"Scenario missing 'id': {scenario}"
            assert "name" in scenario, f"Scenario missing 'name': {scenario}"
            assert "task" in scenario, f"Scenario missing 'task': {scenario}"
            assert "type" in scenario, (
                f"Scenario '{scenario['id']}' missing 'type' field. " f"Must be 'telepathic' or 'brain_action'."
            )
            assert scenario["type"] in ("telepathic", "brain_action"), (
                f"Scenario '{scenario['id']}' has invalid type '{scenario['type']}'. "
                f"Must be 'telepathic' or 'brain_action'."
            )

    def test_brain_action_scenarios_exist(self):
        """CRITICAL: Brain action extraction MUST be benchmarked."""
        with open(SCENARIOS_FILE) as f:
            data = json.load(f)

        brain_scenarios = [s for s in data["scenarios"] if s.get("type") == "brain_action"]
        assert len(brain_scenarios) >= 3, (
            f"Only {len(brain_scenarios)} brain_action scenarios found. "
            f"Need at least 3 to reliably evaluate Brain action extraction."
        )

    def test_brain_scenarios_have_available_actions(self):
        """Brain scenarios must provide available_actions list."""
        with open(SCENARIOS_FILE) as f:
            data = json.load(f)

        for scenario in data["scenarios"]:
            if scenario.get("type") != "brain_action":
                continue
            assert "available_actions" in scenario, f"Brain scenario '{scenario['id']}' missing 'available_actions'"
            assert "target_action" in scenario, f"Brain scenario '{scenario['id']}' missing 'target_action'"
            assert scenario["target_action"] in scenario["available_actions"], (
                f"Brain scenario '{scenario['id']}': target_action "
                f"'{scenario['target_action']}' not in available_actions"
            )

    def test_telepathic_scenarios_have_nodes(self):
        """Telepathic scenarios must provide nodes and target_index."""
        with open(SCENARIOS_FILE) as f:
            data = json.load(f)

        for scenario in data["scenarios"]:
            if scenario.get("type") != "telepathic":
                continue
            assert "nodes" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'nodes'"
            assert "target_index" in scenario, f"Telepathic scenario '{scenario['id']}' missing 'target_index'"


class TestBenchmarkResultsIntegrity:
    """Contract: Stored results must be consistent and comparable."""

    @pytest.fixture
    def results(self):
        if not os.path.exists(RESULTS_FILE):
            pytest.skip("No benchmark results file yet")
        with open(RESULTS_FILE) as f:
            return json.load(f)

    def test_details_format_is_consistent(self, results):
        """All model details must use the same format (object, not raw int)."""
        for model_name, data in results.get("models", {}).items():
            details = data.get("details", {})
            for scenario_id, value in details.items():
                assert isinstance(value, dict), (
                    f"Model '{model_name}' scenario '{scenario_id}' uses "
                    f"legacy format (raw int: {value}). Must be "
                    f"{{'avg_score': int, 'pass_rate': float, 'latency': int}}"
                )

    def test_relative_performance_is_normalized(self, results):
        """Relative performance must not exceed 100% (the leader)."""
        for model_name, data in results.get("models", {}).items():
            pct = data.get("relative_performance_pct", 0)
            assert pct <= 100.0, (
                f"Model '{model_name}' has relative_performance_pct={pct}% > 100%. "
                f"Scoring is not normalized by scenario count!"
            )
