"""Base classes for evaluation runners.

Runners orchestrate the evaluation process: loading benchmarks,
running strategies, computing metrics, and generating reports.
"""

import subprocess
import uuid
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any


def _get_git_commit() -> str:
    """Get current git commit SHA, or 'unknown' if not in a git repo."""
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        return result.stdout.strip()[:12] if result.returncode == 0 else "unknown"
    except Exception:
        return "unknown"


def _is_git_dirty() -> bool:
    """Check if git working directory has uncommitted changes."""
    try:
        result = subprocess.run(
            ["git", "status", "--porcelain"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        return bool(result.stdout.strip()) if result.returncode == 0 else False
    except Exception:
        return False


@dataclass
class RunConfig:
    """Configuration and provenance for an evaluation run.

    Captures all information needed to reproduce an evaluation:
    - Code version (git commit)
    - Model configurations
    - Pricing used for cost calculation
    - Random seeds for reproducibility

    Attributes:
        git_commit: Short SHA of git commit.
        git_dirty: Whether working directory has uncommitted changes.
        python_version: Python version string.
        strategy_config: Strategy configuration dict.
        pricing_snapshot: Model pricing used (model -> (in_cost, out_cost)).
        seeds: Random seeds used for sampling, bootstrap, etc.
        concurrency: Rate limit and concurrency settings.
        notes: Optional human-readable notes.
    """

    git_commit: str = field(default_factory=_get_git_commit)
    git_dirty: bool = field(default_factory=_is_git_dirty)
    python_version: str = ""
    strategy_config: dict[str, Any] = field(default_factory=dict)
    pricing_snapshot: dict[str, tuple[float, float]] = field(default_factory=dict)
    seeds: dict[str, int] = field(default_factory=dict)
    concurrency: dict[str, int] = field(default_factory=dict)
    notes: str = ""

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dict."""
        return {
            "git_commit": self.git_commit,
            "git_dirty": self.git_dirty,
            "python_version": self.python_version,
            "strategy_config": self.strategy_config,
            "pricing_snapshot": {
                k: list(v) for k, v in self.pricing_snapshot.items()
            },
            "seeds": self.seeds,
            "concurrency": self.concurrency,
            "notes": self.notes,
        }


@dataclass
class ExampleResult:
    """Result for a single evaluated example.

    Attributes:
        example_id: ID of the benchmark example.
        strategy_name: Name of the strategy used.
        prediction: Model's output.
        reference: Ground truth answer.
        correct: Whether prediction matches reference.
        score: Numeric quality score (0-1).
        model_used: Which model generated the response.
        tier: Routing tier (if applicable).
        latency_ms: End-to-end latency.
        cost_usd: Estimated cost.
        tokens_in: Input token count.
        tokens_out: Output token count.
        metadata: Example metadata (difficulty, domain, etc.)
        routing_signals: Raw routing signals (mean_logit, etc.)
        error: Error message if evaluation failed.
    """

    example_id: str
    strategy_name: str
    prediction: str
    reference: str | list[str]
    correct: bool
    score: float
    model_used: str
    tier: str | None = None
    latency_ms: float = 0.0
    cost_usd: float = 0.0
    tokens_in: int = 0
    tokens_out: int = 0
    metadata: dict[str, Any] = field(default_factory=dict)
    routing_signals: dict[str, float] = field(default_factory=dict)
    error: str | None = None

    @property
    def difficulty(self) -> str:
        """Get difficulty from metadata."""
        return self.metadata.get("difficulty", "unknown")

    @property
    def domain(self) -> str:
        """Get domain from metadata."""
        return self.metadata.get("domain", "unknown")


@dataclass
class AggregateMetrics:
    """Aggregated metrics with confidence intervals.

    Attributes:
        mean: Mean value.
        std: Standard deviation.
        ci_lower: Lower bound of 95% CI.
        ci_upper: Upper bound of 95% CI.
        n: Sample size.
    """

    mean: float
    std: float = 0.0
    ci_lower: float = 0.0
    ci_upper: float = 0.0
    n: int = 0

    def to_dict(self) -> dict[str, float]:
        return {
            "mean": self.mean,
            "std": self.std,
            "ci_lower": self.ci_lower,
            "ci_upper": self.ci_upper,
            "n": self.n,
        }


@dataclass
class SliceMetrics:
    """Metrics for a specific slice of data.

    Attributes:
        slice_name: Name of this slice (e.g., "hard", "coding").
        slice_value: Value of the slice dimension.
        n_examples: Number of examples in this slice.
        quality: Quality metrics (accuracy, score, etc.)
        efficiency: Efficiency metrics (cost, latency).
        routing: Routing metrics (tier distribution, frac_oracle).
    """

    slice_name: str
    slice_value: str
    n_examples: int
    quality: dict[str, AggregateMetrics] = field(default_factory=dict)
    efficiency: dict[str, AggregateMetrics] = field(default_factory=dict)
    routing: dict[str, Any] = field(default_factory=dict)


@dataclass
class EvaluationResult:
    """Complete result of an evaluation run.

    Attributes:
        evaluation_id: Unique ID for this evaluation.
        strategy_name: Name of the strategy evaluated.
        benchmark_name: Name of the benchmark.
        timestamp: When evaluation was run.
        n_examples: Total number of examples.
        quality: Overall quality metrics.
        efficiency: Overall efficiency metrics.
        routing: Overall routing metrics.
        by_difficulty: Metrics sliced by difficulty.
        by_domain: Metrics sliced by domain.
        by_tier: Metrics sliced by routing tier.
        example_results: Per-example results (for detailed analysis).
        config: Configuration used for this evaluation.
        errors: List of errors encountered.
    """

    evaluation_id: str
    strategy_name: str
    benchmark_name: str
    timestamp: str
    n_examples: int

    # Overall metrics
    quality: dict[str, AggregateMetrics] = field(default_factory=dict)
    efficiency: dict[str, AggregateMetrics] = field(default_factory=dict)
    routing: dict[str, Any] = field(default_factory=dict)

    # Sliced metrics
    by_difficulty: dict[str, SliceMetrics] = field(default_factory=dict)
    by_domain: dict[str, SliceMetrics] = field(default_factory=dict)
    by_tier: dict[str, SliceMetrics] = field(default_factory=dict)

    # Raw data
    example_results: list[ExampleResult] = field(default_factory=list)

    # Metadata
    config: dict[str, Any] = field(default_factory=dict)
    errors: list[str] = field(default_factory=list)

    @classmethod
    def create(
        cls,
        strategy_name: str,
        benchmark_name: str,
    ) -> "EvaluationResult":
        """Create a new EvaluationResult with generated ID and timestamp."""
        return cls(
            evaluation_id=f"eval_{uuid.uuid4().hex[:12]}",
            strategy_name=strategy_name,
            benchmark_name=benchmark_name,
            timestamp=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
            n_examples=0,
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            "evaluation_id": self.evaluation_id,
            "strategy_name": self.strategy_name,
            "benchmark_name": self.benchmark_name,
            "timestamp": self.timestamp,
            "n_examples": self.n_examples,
            "quality": {k: v.to_dict() for k, v in self.quality.items()},
            "efficiency": {k: v.to_dict() for k, v in self.efficiency.items()},
            "routing": self.routing,
            "by_difficulty": {
                k: {
                    "n_examples": v.n_examples,
                    "quality": {qk: qv.to_dict() for qk, qv in v.quality.items()},
                }
                for k, v in self.by_difficulty.items()
            },
            "config": self.config,
            "errors": self.errors,
        }


class Runner(ABC):
    """Abstract base class for evaluation runners.

    Subclasses must implement:
    - run(): Execute the evaluation

    Optionally override:
    - setup(): Pre-evaluation setup
    - teardown(): Post-evaluation cleanup
    """

    @abstractmethod
    async def run(self) -> EvaluationResult:
        """Execute the evaluation.

        Returns:
            EvaluationResult with all metrics and per-example results.
        """

    def setup(self) -> None:  # noqa: B027
        """Pre-evaluation setup. Default is no-op."""
        pass

    def teardown(self) -> None:  # noqa: B027
        """Post-evaluation cleanup. Default is no-op."""
        pass
