"""Error analysis module for understanding evaluation failures.

This module provides tools for categorizing, analyzing, and reporting
evaluation errors. Helps identify systematic failure patterns.
"""

from collections import Counter
from dataclasses import dataclass, field
from enum import Enum
from typing import Any

from embergraphrouter.evals.runners.base import ExampleResult


class ErrorCategory(Enum):
    """Categories of evaluation errors."""

    # Routing errors
    WRONG_MODEL = "wrong_model"  # Router picked suboptimal model
    TIER_MISMATCH = "tier_mismatch"  # Wrong tier classification

    # Model errors
    WRONG_ANSWER = "wrong_answer"  # Model gave incorrect response
    PARTIAL_ANSWER = "partial_answer"  # Partially correct response
    HALLUCINATION = "hallucination"  # Made up information
    REFUSAL = "refusal"  # Model refused to answer

    # Format errors
    PARSE_ERROR = "parse_error"  # Could not parse model output
    FORMAT_ERROR = "format_error"  # Wrong output format

    # System errors
    TIMEOUT = "timeout"  # Request timed out
    RATE_LIMIT = "rate_limit"  # Hit rate limit
    API_ERROR = "api_error"  # API call failed
    UNKNOWN = "unknown"  # Uncategorized error


@dataclass
class ErrorEntry:
    """Single error instance with context.

    Attributes:
        example_id: ID of the failed example.
        category: Error category.
        prediction: Model's output.
        expected: Expected answer.
        error_message: Error description.
        model_used: Model that produced the error.
        metadata: Additional context.
    """

    example_id: str
    category: ErrorCategory
    prediction: str = ""
    expected: str = ""
    error_message: str = ""
    model_used: str = ""
    metadata: dict[str, Any] = field(default_factory=dict)


@dataclass
class ErrorAnalysis:
    """Complete error analysis for an evaluation.

    Attributes:
        total_errors: Total number of errors.
        error_rate: Fraction of examples that errored.
        by_category: Errors grouped by category.
        by_model: Errors grouped by model.
        most_common: Most common error types.
        sample_errors: Sample of error entries for inspection.
        confusion_matrix: Routing confusion matrix if applicable.
    """

    total_errors: int
    error_rate: float
    by_category: dict[ErrorCategory, int]
    by_model: dict[str, int]
    most_common: list[tuple[ErrorCategory, int]]
    sample_errors: list[ErrorEntry]
    confusion_matrix: dict[str, dict[str, int]] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        """Convert to serializable dictionary."""
        return {
            "total_errors": self.total_errors,
            "error_rate": self.error_rate,
            "by_category": {k.value: v for k, v in self.by_category.items()},
            "by_model": self.by_model,
            "most_common": [(cat.value, count) for cat, count in self.most_common],
            "sample_errors": [
                {
                    "example_id": e.example_id,
                    "category": e.category.value,
                    "prediction": e.prediction[:200],  # Truncate
                    "expected": e.expected[:200],
                    "error_message": e.error_message,
                    "model_used": e.model_used,
                }
                for e in self.sample_errors
            ],
            "confusion_matrix": self.confusion_matrix,
        }


def _categorize_error(
    result: ExampleResult,
    oracle_model: str | None = None,
) -> ErrorCategory:
    """Determine error category from example result.

    Args:
        result: Failed example result.
        oracle_model: Best model for this example (if known).

    Returns:
        Appropriate error category.
    """
    # Check for system errors first
    if result.error:
        error_lower = result.error.lower()
        if "timeout" in error_lower:
            return ErrorCategory.TIMEOUT
        if "rate" in error_lower and "limit" in error_lower:
            return ErrorCategory.RATE_LIMIT
        if "api" in error_lower or "connection" in error_lower:
            return ErrorCategory.API_ERROR

    # Check prediction characteristics
    prediction = result.prediction.lower().strip()

    # Refusal patterns
    refusal_patterns = [
        "i cannot",
        "i can't",
        "i'm not able to",
        "i apologize",
        "as an ai",
        "i don't have access",
    ]
    if any(pattern in prediction for pattern in refusal_patterns):
        return ErrorCategory.REFUSAL

    # Parse error (empty or garbled output)
    if not prediction or len(prediction) < 3:
        return ErrorCategory.PARSE_ERROR

    # Routing errors
    if oracle_model and result.model_used != oracle_model:
        return ErrorCategory.WRONG_MODEL

    # Default to wrong answer
    if 0 < result.score < 1.0:
        return ErrorCategory.PARTIAL_ANSWER

    return ErrorCategory.WRONG_ANSWER


def _build_confusion_matrix(
    results: list[ExampleResult],
    oracle_models: dict[str, str] | None = None,
) -> dict[str, dict[str, int]]:
    """Build routing confusion matrix.

    Args:
        results: List of example results.
        oracle_models: Mapping of example_id to oracle model.

    Returns:
        Confusion matrix: predicted -> actual -> count.
    """
    if not oracle_models:
        return {}

    matrix: dict[str, dict[str, int]] = {}

    for result in results:
        predicted = result.model_used or "unknown"
        actual = oracle_models.get(result.example_id, "unknown")

        if predicted not in matrix:
            matrix[predicted] = {}
        if actual not in matrix[predicted]:
            matrix[predicted][actual] = 0
        matrix[predicted][actual] += 1

    return matrix


def analyze_errors(
    results: list[ExampleResult],
    oracle_models: dict[str, str] | None = None,
    max_samples: int = 20,
) -> ErrorAnalysis:
    """Analyze errors from evaluation results.

    Args:
        results: List of example results (includes both correct and incorrect).
        oracle_models: Optional mapping of example_id to best model.
        max_samples: Maximum error samples to include.

    Returns:
        ErrorAnalysis with categorized errors and statistics.
    """
    # Filter to errors only
    errors = [r for r in results if not r.correct]

    if not errors:
        return ErrorAnalysis(
            total_errors=0,
            error_rate=0.0,
            by_category={},
            by_model={},
            most_common=[],
            sample_errors=[],
            confusion_matrix={},
        )

    # Categorize each error
    error_entries: list[ErrorEntry] = []
    category_counts: Counter[ErrorCategory] = Counter()
    model_counts: Counter[str] = Counter()

    for result in errors:
        oracle = oracle_models.get(result.example_id) if oracle_models else None
        category = _categorize_error(result, oracle)

        # Get expected as string (reference may be str or list[str])
        expected = result.reference
        if isinstance(expected, list):
            expected = expected[0] if expected else ""

        entry = ErrorEntry(
            example_id=result.example_id,
            category=category,
            prediction=result.prediction,
            expected=expected,
            error_message=result.error or "",
            model_used=result.model_used or "unknown",
            metadata=result.metadata,
        )
        error_entries.append(entry)
        category_counts[category] += 1
        model_counts[entry.model_used] += 1

    # Build confusion matrix
    confusion = _build_confusion_matrix(results, oracle_models)

    # Sort by most common
    most_common = category_counts.most_common()

    return ErrorAnalysis(
        total_errors=len(errors),
        error_rate=len(errors) / len(results) if results else 0.0,
        by_category=dict(category_counts),
        by_model=dict(model_counts),
        most_common=most_common,
        sample_errors=error_entries[:max_samples],
        confusion_matrix=confusion,
    )


def summarize_errors(analysis: ErrorAnalysis) -> str:
    """Generate human-readable error summary.

    Args:
        analysis: Error analysis results.

    Returns:
        Formatted error summary string.
    """
    if analysis.total_errors == 0:
        return "No errors found."

    lines = [
        "Error Summary",
        "=============",
        f"Total errors: {analysis.total_errors} ({analysis.error_rate:.1%})",
        "",
        "By Category:",
    ]

    for category, count in analysis.most_common[:5]:
        pct = count / analysis.total_errors * 100
        lines.append(f"  - {category.value}: {count} ({pct:.1f}%)")

    if analysis.by_model:
        lines.append("")
        lines.append("By Model:")
        for model, count in sorted(
            analysis.by_model.items(), key=lambda x: -x[1]
        )[:5]:
            pct = count / analysis.total_errors * 100
            lines.append(f"  - {model}: {count} ({pct:.1f}%)")

    return "\n".join(lines)


__all__ = [
    "ErrorAnalysis",
    "ErrorCategory",
    "ErrorEntry",
    "analyze_errors",
    "summarize_errors",
]
