"""
Golden dataset regression test runner.

Loads curated educational content items from golden_dataset.json,
re-evaluates each one, and compares the new verdict against the
expected verdict to detect regressions.

Enhanced to capture detailed per-metric results for failure analysis.
"""

import asyncio
import json
import logging
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

from ..service import EvaluationService
from ..core.input_models import RequestMetadata

logger = logging.getLogger(__name__)

GOLDEN_DATASET_PATH = Path(__file__).parent / "golden_dataset.json"

# Verdicts that count as "passing" quality
_PASS_VERDICTS = {"ACCEPTABLE", "SUPERIOR"}

# Metrics that appear on all content types
_UNIVERSAL_METRICS = [
    "factual_accuracy",
    "educational_effectiveness",
    "localization_quality",
]

# Content-type-specific metrics
_CONTENT_TYPE_METRICS = {
    "instructional_question": ["clarity", "difficulty_alignment", "distractor_quality", "stimulus_quality"],
    "assessment_question": ["clarity", "difficulty_alignment", "distractor_quality", "stimulus_quality"],
    "quiz": ["concept_coverage", "difficulty_distribution", "non_repetitiveness", "answer_balance"],
    "article": ["clarity", "follows_direct_instruction"],
    "reading_passage": ["reading_level_match", "text_quality"],
    "other": ["clarity"],
}


def _enum_to_str(val) -> str:
    """Safely convert an enum or string value to a plain string."""
    return val.value if hasattr(val, 'value') else str(val) if val else ""


@dataclass
class MetricDetail:
    """Detailed result for a single metric."""
    name: str
    score: float
    reasoning: str = ""
    suggested_improvements: str = ""


@dataclass
class GoldenTestResult:
    """Result of a single golden dataset item evaluation."""

    item_id: str
    expected_verdict: str
    actual_verdict: str
    actual_score: float
    passed: bool  # True if expected == actual verdict category
    error: Optional[str] = None
    # Detailed metric results
    content_type: str = ""
    metrics: list[MetricDetail] = field(default_factory=list)
    overall_reasoning: str = ""
    overall_improvements: str = ""
    # Item metadata for context
    metadata: dict = field(default_factory=dict)


@dataclass
class GoldenTestReport:
    """Aggregated report for the full golden dataset run."""

    total: int
    passed: int
    failed: int
    errors: int
    pass_rate: float
    threshold: float
    threshold_met: bool
    results: list[GoldenTestResult] = field(default_factory=list)
    duration_seconds: float = 0.0

    @property
    def regressions(self) -> list[GoldenTestResult]:
        """Return items that regressed (wrong verdict, excluding errors)."""
        return [r for r in self.results if not r.passed and r.error is None]


def _load_golden_dataset() -> dict:
    """Load and return the golden dataset JSON."""
    with open(GOLDEN_DATASET_PATH, "r", encoding="utf-8") as f:
        return json.load(f)


def _content_to_string(generated_content) -> str:
    """Convert generated_content to a string suitable for evaluation."""
    if isinstance(generated_content, dict):
        return json.dumps(generated_content, indent=2)
    return str(generated_content)


def _build_request_metadata(raw_metadata: Optional[dict]) -> Optional[RequestMetadata]:
    """Build a RequestMetadata instance from a raw dict, or None."""
    if raw_metadata is None:
        return None
    return RequestMetadata(**raw_metadata)


def _verdicts_match(expected: str, actual: str) -> bool:
    """
    Compare expected and actual verdicts by category.

    Both ACCEPTABLE and SUPERIOR count as the "pass" category.
    INFERIOR is the "fail" category.
    """
    expected_passes = expected.upper() in _PASS_VERDICTS
    actual_passes = actual.upper() in _PASS_VERDICTS
    return expected_passes == actual_passes


def _extract_metrics(result) -> tuple[list[MetricDetail], str, str]:
    """Extract all metric details from an evaluation result."""
    metrics = []
    content_type = getattr(result, 'content_type', None)
    content_type_str = _enum_to_str(content_type)

    metric_names = list(_UNIVERSAL_METRICS)
    if content_type_str in _CONTENT_TYPE_METRICS:
        metric_names.extend(_CONTENT_TYPE_METRICS[content_type_str])

    for metric_name in metric_names:
        metric_obj = getattr(result, metric_name, None)
        if metric_obj is not None and hasattr(metric_obj, 'score'):
            metrics.append(MetricDetail(
                name=metric_name,
                score=metric_obj.score,
                reasoning=getattr(metric_obj, 'reasoning', '') or '',
                suggested_improvements=getattr(metric_obj, 'suggested_improvements', '') or '',
            ))

    overall_obj = getattr(result, 'overall', None)
    overall_reasoning = ''
    overall_improvements = ''
    if overall_obj:
        overall_reasoning = getattr(overall_obj, 'reasoning', '') or ''
        overall_improvements = getattr(overall_obj, 'suggested_improvements', '') or ''

    return metrics, overall_reasoning, overall_improvements


async def _evaluate_single_item(
    item: dict,
    service: EvaluationService,
    semaphore: asyncio.Semaphore,
) -> GoldenTestResult:
    """Evaluate a single golden dataset item under semaphore control."""
    item_id = item.get("id", "unknown")
    expected_verdict = item.get("expected_verdict", "UNKNOWN")
    item_metadata = item.get("metadata", {})

    async with semaphore:
        try:
            # Reconstruct the content string
            content_str = _content_to_string(item.get("generated_content", ""))

            # Build request metadata if present
            request_metadata = _build_request_metadata(
                item.get("request_metadata")
            )

            # Extract curriculum from metadata
            curriculum = item.get("metadata", {}).get("curriculum")

            # Run evaluation
            result = await service.evaluate(
                content=content_str,
                request_metadata=request_metadata,
                curriculum=curriculum,
            )

            actual_verdict = _enum_to_str(result.overall_rating)
            actual_score = result.overall.score
            passed = _verdicts_match(expected_verdict, actual_verdict)

            # Extract detailed metrics
            content_type = _enum_to_str(result.content_type)
            metrics, overall_reasoning, overall_improvements = _extract_metrics(result)

            return GoldenTestResult(
                item_id=item_id,
                expected_verdict=expected_verdict,
                actual_verdict=actual_verdict,
                actual_score=actual_score,
                passed=passed,
                content_type=content_type,
                metrics=metrics,
                overall_reasoning=overall_reasoning,
                overall_improvements=overall_improvements,
                metadata=item_metadata,
            )

        except Exception as e:
            logger.error(f"Error evaluating golden item {item_id}: {e}")
            return GoldenTestResult(
                item_id=item_id,
                expected_verdict=expected_verdict,
                actual_verdict="ERROR",
                actual_score=0.0,
                passed=False,
                error=str(e),
                metadata=item_metadata,
            )


async def run_golden_test(
    threshold: float = 0.98,
    max_concurrent: int = 5,
    verbose: bool = False,
    on_progress: Optional[callable] = None,
) -> GoldenTestReport:
    """
    Run the golden dataset regression test.

    Loads every item from golden_dataset.json, re-evaluates it, and
    compares the new overall_rating against the expected_verdict.

    Args:
        threshold: Minimum pass rate (0.0-1.0) to consider the test passing.
        max_concurrent: Maximum number of concurrent evaluations.
        verbose: If True, log extra detail per item.
        on_progress: Optional callback(completed, total) called after each item.

    Returns:
        GoldenTestReport with full results and aggregate statistics.
    """
    dataset = _load_golden_dataset()
    items = dataset.get("items", [])
    total = len(items)

    if total == 0:
        return GoldenTestReport(
            total=0,
            passed=0,
            failed=0,
            errors=0,
            pass_rate=1.0,
            threshold=threshold,
            threshold_met=True,
        )

    service = EvaluationService()
    semaphore = asyncio.Semaphore(max_concurrent)
    start_time = time.time()

    # Track progress with a mutable counter
    completed = [0]

    async def _run_and_track(item: dict) -> GoldenTestResult:
        result = await _evaluate_single_item(item, service, semaphore)
        completed[0] += 1
        if verbose:
            status = "PASS" if result.passed else ("ERROR" if result.error else "FAIL")
            logger.info(
                f"[{completed[0]}/{total}] {result.item_id}: {status} "
                f"(expected={result.expected_verdict}, actual={result.actual_verdict})"
            )
        if on_progress is not None:
            on_progress(completed[0], total)
        return result

    # Run all evaluations concurrently (bounded by semaphore)
    tasks = [_run_and_track(item) for item in items]
    results = await asyncio.gather(*tasks)

    duration = time.time() - start_time

    # Aggregate statistics
    passed = sum(1 for r in results if r.passed)
    errors = sum(1 for r in results if r.error is not None)
    failed = total - passed - errors
    evaluated = total - errors
    pass_rate = (passed / evaluated) if evaluated > 0 else 0.0

    return GoldenTestReport(
        total=total,
        passed=passed,
        failed=failed,
        errors=errors,
        pass_rate=pass_rate,
        threshold=threshold,
        threshold_met=pass_rate >= threshold,
        results=list(results),
        duration_seconds=round(duration, 2),
    )
