"""Evaluation harness for fine-tuned ModernBERT routing classifiers.

``RouterEvaluator`` measures accuracy against a labelled test set and compares
against the generic (unfine-tuned) ModernBERT baseline.  A quality gate
(``passes_gate``) enforces minimum improvement and latency requirements before
newly trained weights are considered safe to promote to production.

Quality gate criteria:
- ``improvement_pp >= 5.0`` — at least 5 percentage-point accuracy gain
- ``latency_p50_ms < 50`` — median inference latency under 50 ms on CPU

Privacy note: Evaluation uses synthetic examples only.
"""

from __future__ import annotations

import dataclasses
import logging
import re
import time
from typing import TYPE_CHECKING, Any

from llmhosts.training.curator import COMPLEXITY_LABELS as _COMPLEXITY_LABELS

if TYPE_CHECKING:
    from pathlib import Path

    from llmhosts.training.curator import TrainingExample

logger = logging.getLogger(__name__)


@dataclasses.dataclass
class EvalResult:
    """Evaluation result comparing a fine-tuned model to baseline."""

    accuracy: float
    """Macro-averaged accuracy across complexity + domain + quality heads."""

    improvement_pp: float
    """Accuracy improvement in percentage points vs baseline."""

    latency_p50_ms: float
    """50th percentile (median) inference latency in milliseconds."""

    latency_p99_ms: float
    """99th percentile inference latency in milliseconds."""

    passes_gate: bool
    """True when improvement_pp >= 5.0 AND latency_p50_ms < 50."""

    n_samples: int
    """Number of evaluation examples used."""


class RouterEvaluator:
    """Evaluates a fine-tuned ModernBERT adapter against a labelled test set.

    Usage::

        evaluator = RouterEvaluator()
        baseline_acc = evaluator.evaluate_baseline(examples)
        result = evaluator.evaluate(model_path, examples, baseline_accuracy=baseline_acc)
        if result.passes_gate:
            registry.activate(version)
    """

    _GATE_IMPROVEMENT_PP: float = 5.0
    _GATE_LATENCY_P50_MS: float = 50.0

    def evaluate(
        self,
        model_path: Path,
        examples: list[TrainingExample],
        baseline_accuracy: float = 0.72,
    ) -> EvalResult:
        """Evaluate a fine-tuned adapter on the provided examples.

        Loads the LoRA adapter from ``model_path`` and runs inference on
        ``examples``.  Computes accuracy, latency percentiles, and applies
        the quality gate.

        Args:
            model_path: Path to the saved LoRA adapter directory.
            examples: Labelled evaluation examples.
            baseline_accuracy: Accuracy of the unfine-tuned baseline
                (from ``evaluate_baseline``).  Defaults to 0.72 which matches
                the heuristic classifier's typical accuracy.

        Returns:
            ``EvalResult`` with accuracy, latency, and gate status.
        """
        if not examples:
            return EvalResult(
                accuracy=0.0,
                improvement_pp=0.0,
                latency_p50_ms=0.0,
                latency_p99_ms=0.0,
                passes_gate=False,
                n_samples=0,
            )

        try:
            import torch
            from peft import PeftModel
            from transformers import AutoModelForSequenceClassification, AutoTokenizer

            base_model_name = _detect_base_model(model_path)
            tokenizer = AutoTokenizer.from_pretrained(base_model_name)  # nosec B615
            base_model = AutoModelForSequenceClassification.from_pretrained(  # nosec B615
                base_model_name,
                num_labels=len(_COMPLEXITY_LABELS),
                ignore_mismatched_sizes=True,
            )
            model = PeftModel.from_pretrained(base_model, str(model_path))
            model.eval()

            correct = 0
            latencies: list[float] = []

            complexity_to_id = {label: i for i, label in enumerate(_COMPLEXITY_LABELS)}

            with torch.no_grad():
                for ex in examples:
                    inputs = tokenizer(
                        ex.text,
                        return_tensors="pt",
                        padding=True,
                        truncation=True,
                        max_length=512,
                    )
                    t0 = time.perf_counter()
                    outputs = model(**inputs)
                    latencies.append((time.perf_counter() - t0) * 1000)

                    pred_id = outputs.logits.argmax(dim=-1).item()
                    expected_id = complexity_to_id.get(ex.complexity_label, -1)
                    if pred_id == expected_id:
                        correct += 1

            accuracy = correct / len(examples)

        except ImportError:
            logger.warning("PEFT/transformers not available; using heuristic accuracy estimate for evaluation.")
            accuracy = self._heuristic_accuracy(examples)
            latencies = [10.0] * len(examples)  # Heuristic is fast

        except Exception as exc:
            logger.error("Evaluation failed: %s", exc)
            accuracy = 0.0
            latencies = [0.0]

        latencies_sorted = sorted(latencies)
        n = len(latencies_sorted)
        p50 = latencies_sorted[n // 2] if n > 0 else 0.0
        p99_idx = int(n * 0.99) if n > 1 else n - 1
        p99 = latencies_sorted[min(p99_idx, n - 1)] if n > 0 else 0.0

        improvement_pp = round((accuracy - baseline_accuracy) * 100, 2)
        passes_gate = improvement_pp >= self._GATE_IMPROVEMENT_PP and p50 < self._GATE_LATENCY_P50_MS

        return EvalResult(
            accuracy=round(accuracy, 4),
            improvement_pp=improvement_pp,
            latency_p50_ms=round(p50, 2),
            latency_p99_ms=round(p99, 2),
            passes_gate=passes_gate,
            n_samples=len(examples),
        )

    def evaluate_baseline(self, examples: list[TrainingExample]) -> float:
        """Evaluate the existing heuristic ModernBERT classifier on examples.

        Runs the hand-crafted heuristic classifier (always available, no GPU)
        against the labelled examples and returns accuracy.

        Args:
            examples: Labelled evaluation examples.

        Returns:
            Fraction of examples correctly classified (complexity head only).
        """
        if not examples:
            return 0.0
        return self._heuristic_accuracy(examples)

    # ------------------------------------------------------------------
    # Internal helpers
    # ------------------------------------------------------------------

    def _heuristic_classify_complexity(self, text: str) -> str:
        """Instance wrapper around the module-level heuristic classifier."""
        return _heuristic_classify_complexity(text)

    @staticmethod
    def _heuristic_accuracy(examples: list[TrainingExample]) -> float:
        """Estimate accuracy using simple heuristic rules (no model required).

        This approximates the ModernBERT heuristic classifier's behaviour
        without requiring the full router stack.  Used when PEFT is unavailable
        or for quick sanity checks.
        """
        if not examples:
            return 0.0

        correct = 0
        for ex in examples:
            predicted = _heuristic_classify_complexity(ex.text)
            if predicted == ex.complexity_label:
                correct += 1

        return correct / len(examples)


def _heuristic_classify_complexity(text: str) -> str:
    """Lightweight heuristic complexity classifier for evaluation purposes."""
    text_lower = text.lower()
    n_words = len(text.split())

    # Expert indicators
    expert_patterns = [
        "byzantine",
        "raft",
        "consensus",
        "jit compiler",
        "crdt",
        "zero-downtime",
        "lock-free",
        "formal verification",
        "phd",
        "ieee conference",
        "peer-reviewed",
    ]
    if any(p in text_lower for p in expert_patterns):
        return "expert"

    # Complex indicators
    complex_patterns = [
        "distributed",
        "multi-tenant",
        "oauth 2.0",
        "concurrent",
        "architecture",
        "comprehensive",
        "detailed design",
        "whitepaper",
        "post-mortem",
        "trade-off analysis",
    ]
    complexity_word_pattern = re.compile(
        r"\b(?:step by step|multi-step|compare and contrast|synthesize|nuanced)\b",
        re.IGNORECASE,
    )
    if any(p in text_lower for p in complex_patterns) or complexity_word_pattern.search(text):
        return "complex"

    # Simple / trivial indicators
    if n_words <= 5:
        return "trivial"
    if n_words <= 15 and not any(p in text_lower for p in ["implement", "design", "analyse", "review"]):
        return "simple"

    return "moderate"


def _detect_base_model(adapter_path: Path) -> str:
    """Read the base model name from the adapter's config if present."""
    import json

    config_file = adapter_path / "adapter_config.json"
    if config_file.exists():
        try:
            data = json.loads(config_file.read_text())
            base: Any = data.get("base_model_name_or_path", "")
            if base and isinstance(base, str):
                return str(base)
        except Exception:
            pass
    return "answerdotai/ModernBERT-base"
