"""Evaluation engine — runs metrics against test cases."""

from __future__ import annotations

import asyncio
import logging
from concurrent.futures import ThreadPoolExecutor

from evalkit.judges.base import BaseJudge
from evalkit.metrics.base import BaseMetric
from evalkit.models import EvaluationReport, EvaluationResult, TestCase
from evalkit.reports.generator import ReportGenerator
from evalkit.suites.base import EvaluationSuite

logger = logging.getLogger(__name__)


class EvaluationEngine:
    """Orchestrates running evaluation metrics against test cases."""

    def __init__(self, judge: BaseJudge | None = None, concurrency: int = 4) -> None:
        self.judge = judge
        self.concurrency = concurrency
        self._report_generator = ReportGenerator()

    def run(
        self,
        test_cases: list[TestCase],
        suite: EvaluationSuite | None = None,
        metrics: list[BaseMetric] | None = None,
    ) -> EvaluationReport:
        """Run evaluation synchronously. Provide either *suite* or *metrics*."""
        resolved_metrics, suite_name = self._resolve_metrics(suite, metrics)
        metric_names = [m.name for m in resolved_metrics]
        logger.info(
            "Starting sync evaluation: suite=%s, metrics=%s, test_cases=%d, concurrency=%d",
            suite_name,
            metric_names,
            len(test_cases),
            self.concurrency,
        )

        with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
            futures = [executor.submit(self._score_test_case, tc, resolved_metrics) for tc in test_cases]
            results: list[EvaluationResult] = [f.result() for f in futures]

        report = self._report_generator.generate(results, suite_name, metrics=resolved_metrics, judge=self.judge)
        logger.info(
            "Evaluation complete: average_score=%.3f, passed=%d/%d",
            report.summary.average_score,
            report.summary.passed,
            report.summary.total_cases,
        )
        return report

    async def run_async(
        self,
        test_cases: list[TestCase],
        suite: EvaluationSuite | None = None,
        metrics: list[BaseMetric] | None = None,
    ) -> EvaluationReport:
        """Run evaluation asynchronously. Provide either *suite* or *metrics*."""
        resolved_metrics, suite_name = self._resolve_metrics(suite, metrics)
        metric_names = [m.name for m in resolved_metrics]
        logger.info(
            "Starting async evaluation: suite=%s, metrics=%s, test_cases=%d, concurrency=%d",
            suite_name,
            metric_names,
            len(test_cases),
            self.concurrency,
        )
        semaphore = asyncio.Semaphore(self.concurrency)

        results: list[EvaluationResult] = await asyncio.gather(
            *[self._score_test_case_async(tc, resolved_metrics, semaphore) for tc in test_cases]
        )

        report = self._report_generator.generate(list(results), suite_name, metrics=resolved_metrics, judge=self.judge)
        logger.info(
            "Evaluation complete: average_score=%.3f, passed=%d/%d",
            report.summary.average_score,
            report.summary.passed,
            report.summary.total_cases,
        )
        return report

    def _resolve_metrics(
        self,
        suite: EvaluationSuite | None,
        metrics: list[BaseMetric] | None,
    ) -> tuple[list[BaseMetric], str]:
        if suite is not None and metrics is not None:
            raise ValueError("Provide either suite or metrics, not both")
        if suite is None and metrics is None:
            raise ValueError("Must provide either suite or metrics")
        if suite is not None:
            return suite.metrics, suite.name
        assert metrics is not None
        return metrics, "custom"

    def _score_test_case(
        self,
        test_case: TestCase,
        metrics: list[BaseMetric],
    ) -> EvaluationResult:
        logger.debug("Scoring test case: %s", test_case.question[:80])
        scores = [metric.score(test_case, self.judge) for metric in metrics]
        overall = sum(s.score for s in scores) / len(scores) if scores else 0.0
        logger.debug("Test case scored: overall=%.3f, question=%s", overall, test_case.question[:80])
        return EvaluationResult(test_case=test_case, scores=scores)

    async def _score_test_case_async(
        self,
        test_case: TestCase,
        metrics: list[BaseMetric],
        semaphore: asyncio.Semaphore,
    ) -> EvaluationResult:
        logger.debug("Scoring test case (async): %s", test_case.question[:80])
        async with semaphore:
            scores = await asyncio.gather(*[metric.score_async(test_case, self.judge) for metric in metrics])
        overall = sum(s.score for s in scores) / len(scores) if scores else 0.0
        logger.debug("Test case scored (async): overall=%.3f, question=%s", overall, test_case.question[:80])
        return EvaluationResult(test_case=test_case, scores=list(scores))
