"""Two-tier quality evaluator for netanel-core.

Evaluates LLM outputs using a progressive quality ratchet.
Tier 1: gpt-4o-mini (cheap, handles ~95% of evaluations)
Tier 2: gpt-4o (only when Tier 1 is uncertain — borderline score or low confidence)

The quality threshold starts low (0.7) and auto-raises as the system improves,
capping at 0.95. This prevents endless retry loops early on while pushing
quality upward over time.

Architecture ref: ARCHITECTURE.md Section 6 (Complete Flow, steps 4-5)
"""

from __future__ import annotations

import json
import logging
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol

from filelock import FileLock
from pydantic import BaseModel, ConfigDict, Field

from netanel_core._atomic import atomic_write_json
from netanel_core.config import NathanConfig

logger = logging.getLogger(__name__)


# ────────────────────────────────────────────
# Score Utilities
# ────────────────────────────────────────────


def _clamp_score(score: float) -> float:
    """Clamp a score to [0.0, 1.0], handling NaN and Inf (BUG-061).

    Python's ``max(0.0, min(1.0, float('nan')))`` evaluates to ``nan``
    because NaN comparisons always return False.  Check for non-finite
    values first and return 0.0 as a safe default before clamping.

    Args:
        score: Raw float from the LLM structured output.

    Returns:
        A finite float in [0.0, 1.0].
    """
    if math.isnan(score) or math.isinf(score):
        return 0.0
    return max(0.0, min(1.0, score))


# ────────────────────────────────────────────
# Action Task Heuristic
# ────────────────────────────────────────────

_ACTION_KEYWORDS: frozenset[str] = frozenset({
    "write", "writing",
    "create", "creating",
    "delete", "deleting",
    "remove", "removing",
    "run",
    "execute", "executing",
    "save", "saving",
    "update", "updating",
    "move", "moving",
    "copy",
})


def _is_action_task(task: str) -> bool:
    """Return True if the task description appears to involve a system action.

    Heuristic: checks for action keywords and their gerund (-ing) forms using
    substring matching against the lowercased task string. Matched keywords:
    write/writing, create/creating, delete/deleting, remove/removing, run,
    execute/executing, save/saving, update/updating, move/moving, copy.
    Used to decide whether to include a conservative scoring note when
    tool_results are absent.

    Args:
        task: The task description string.

    Returns:
        True if any action keyword is found in the task.
    """
    lowered = task.lower()
    return any(kw in lowered for kw in _ACTION_KEYWORDS)


# ────────────────────────────────────────────
# Evaluation Prompt
# ────────────────────────────────────────────

_EVAL_SYSTEM_PROMPT = """\
You are a strict quality evaluator for an AI system. Score LLM outputs rigorously. \
Your default assumption is that every output has room for improvement.

## Scoring Criteria (each 0.0-1.0, overall = average)
- **Completeness**: Every requirement addressed? Missing ANY part caps this at <=0.8.
- **Correctness**: Factually and technically accurate? Any error caps this at <=0.7.
- **Clarity**: Well-structured, concise, easy to follow?
- **Relevance**: Focused on the task? No padding or off-topic content?

## Score Anchors
- **1.0** -- Flawless. Nothing to improve. Handles every edge case. Extremely rare.
- **0.9** -- Excellent. Minor polish only. No functional gaps.
- **0.8** -- Good. One small gap or unclear section.
- **0.7** -- Adequate. Noticeable gaps or one significant issue.
- **0.5** -- Mediocre. Major gaps or multiple errors.
- **0.0-0.4** -- Poor. Fails to address the task or has critical errors.

## Before You Score (MANDATORY)
1. Actively search for gaps: What is missing? What could break? What is ambiguous?
2. Check edge cases: Does the output handle boundary conditions?
3. If the output is code: Is error handling present? Are inputs validated?
4. If you score above 0.9: Explicitly state what makes this nearly or fully perfect.
5. A score of 1.0 MUST be justified -- "it looks good" is not justification.

## Rules
1. Score each criterion 0.0 to 1.0
2. Overall score = average of all four criteria
3. Rate your confidence in this evaluation (0.0 to 1.0)
4. Provide specific, actionable feedback -- even for high scores, name what would make it perfect
5. Be calibrated: a solid response earns 0.8, excellent earns 0.9, flawless earns 1.0

START DIRECTLY with the evaluation. No preambles."""


def _build_eval_prompt(
    task: str,
    output: str,
    context: str | None = None,
    *,
    tool_results: list | None = None,
) -> str:
    """Build the human message for evaluation.

    Assembles task, output, optional context, and optional tool results into
    a structured prompt for the evaluation model. When tool_results are provided
    they are appended so the LLM evaluator can verify claims against reality.
    When absent for action tasks, a conservative scoring note is added.

    Args:
        task: The original task/instruction.
        output: The LLM output to evaluate.
        context: Optional additional context for the evaluator.
        tool_results: Optional list of actual tool execution results. When
            provided, the evaluator can check the agent's claims against
            what really happened. When None and the task is an action task,
            a conservative note warns the evaluator it is operating blind.

    Returns:
        Formatted evaluation prompt string.
    """
    parts = [f"## Task\n{task}", f"## Output to Evaluate\n{output}"]
    if context:
        parts.append(f"## Additional Context\n{context}")
    if tool_results is not None:
        # Format each tool result as a numbered entry for readability
        formatted = "\n".join(
            f"{i + 1}. {result}" for i, result in enumerate(tool_results)
        )
        parts.append(
            f"## Actual Execution Results\n"
            f"The following tool results show what ACTUALLY happened during execution. "
            f"Use these to verify the agent's claims.\n\n{formatted}"
        )
    elif _is_action_task(task):
        parts.append(
            "## Evaluation Notice\n"
            "Note: No execution results were provided — this is a text-only assessment. "
            "Be conservative when scoring tasks that claim to have performed file or "
            "system operations. The agent's statements about what it did cannot be "
            "verified from the output alone."
        )
    return "\n\n".join(parts)



# ────────────────────────────────────────────
# Data Models
# ────────────────────────────────────────────


class EvalResponse(BaseModel):
    """Structured response from the evaluation LLM call.

    Returned by the LLM via structured output (function calling).
    """

    model_config = ConfigDict(extra="forbid")

    score: float = Field(
        ge=0.0,
        le=1.0,
        description="Overall quality score (average of completeness, correctness, clarity, relevance)",
    )
    confidence: float = Field(
        ge=0.0,
        le=1.0,
        description="How confident the evaluator is in this score",
    )
    reasoning: str = Field(
        description="Explanation of why this score was given",
    )
    feedback: str = Field(
        description="Specific, actionable suggestions for improvement",
    )


@dataclass
class EvalResult:
    """Result of evaluating an LLM output.

    Returned by Evaluator.evaluate() — consumed by LearningLLM, PromptEvolver, DeepAgent.
    """

    score: float
    """Quality score from 0.0 to 1.0."""

    passed: bool
    """Whether the score meets or exceeds the current threshold."""

    feedback: str
    """Actionable improvement suggestions from the evaluator."""

    reasoning: str
    """Explanation of why this score was given."""

    tier_used: int
    """Which evaluation tier produced this result (1 or 2)."""

    threshold: float
    """The threshold that was applied for this evaluation."""

    confidence: float
    """Evaluator's confidence in the score (0.0 to 1.0)."""

    unverified: bool = False
    """True when tool_results were not provided to evaluate().

    An unverified result means the evaluator had no access to actual execution
    results and could only assess the agent's text claims. Scores for action
    tasks (write, create, delete, run, execute) are particularly suspect when
    unverified — the agent may be hallucinating success.

    Set to False when tool_results is an empty list (tool was called but returned
    nothing — that is a verified empty result, not a missing result).
    """


class EvalState(BaseModel):
    """Persistent state for the progressive quality ratchet.

    Saved to eval_state.json in the namespace directory.
    Tracks score history and threshold progression.
    """

    model_config = ConfigDict(extra="forbid")

    # Default 0.7 is a fallback only. Evaluator._load_state() always overrides
    # this from EvalConfig.initial_threshold when creating fresh state.
    current_threshold: float = Field(
        default=0.7,
        description="Current quality threshold (starts low, auto-raises)",
    )
    scores_history: list[float] = Field(
        default_factory=list,
        description="Recent evaluation scores for rolling average",
    )
    total_evaluations: int = Field(
        default=0,
        description="Total number of evaluations performed",
    )
    tier2_escalations: int = Field(
        default=0,
        description="Number of times Tier 2 was invoked",
    )
    threshold_raises: int = Field(
        default=0,
        description="Number of times the threshold was auto-raised",
    )


# ────────────────────────────────────────────
# Evaluator Model Protocol
# ────────────────────────────────────────────


class EvalModel(Protocol):
    """Protocol for evaluation models — enables dependency injection for testing."""

    def invoke(self, messages: list[Any]) -> EvalResponse: ...


# Asymmetric escalation: passing scores don't need Tier 2 validation (BUG-018).
# 0.0 = no escalation above threshold. Lower half keeps full uncertainty_range coverage.
# Named constant (not config) — most users won't need to tune this.
_ESCALATION_ABOVE_THRESHOLD_RATIO: float = 0.0


# ────────────────────────────────────────────
# Evaluator
# ────────────────────────────────────────────


class Evaluator:
    """Two-tier quality evaluator with progressive ratchet.

    Evaluates LLM outputs against a dynamic quality threshold that
    starts low and auto-raises as the system improves.

    Tier 1 (gpt-4o-mini): Handles ~95% of evaluations. Cheap and fast.
    Tier 2 (gpt-4o): Only invoked when Tier 1 score is borderline or
    confidence is low. More capable but more expensive.

    Usage:
        evaluator = Evaluator(config)
        result = evaluator.evaluate("Write a sort function", "def sort(arr): ...")
        if not result.passed:
            # retry with result.feedback
    """

    # TODO: Implement calibration phase using EvalConfig.calibration_calls (default: 50).
    # During the first `calibration_calls` evaluations, run BOTH Tier 1 and Tier 2
    # on every call. Compare scores to build a calibration offset so Tier 1 can
    # match Tier 2 accuracy in production. Track in EvalState.

    def __init__(
        self,
        config: NathanConfig,
        tier1_model: EvalModel | None = None,
        tier2_model: EvalModel | None = None,
    ) -> None:
        """Initialize the evaluator.

        Args:
            config: Nathan configuration with evaluation settings.
            tier1_model: Override Tier 1 model (for testing). If None, creates
                ChatOpenAI with config.models.evaluator_model.
            tier2_model: Override Tier 2 model (for testing). If None, creates
                ChatOpenAI with config.models.evaluator_tier2_model.
        """
        self._config = config
        self._state = self._load_state()

        if tier1_model is not None:
            self._tier1 = tier1_model
        else:
            from langchain_openai import ChatOpenAI

            # LangChain Runnable type is broader than our EvalModel Protocol,
            # but invoke() works correctly at runtime.
            self._tier1 = ChatOpenAI(  # type: ignore[assignment]
                model=config.models.evaluator_model,
                temperature=0,
                timeout=config.safety.model_timeout_seconds,
            ).with_structured_output(EvalResponse)

        if tier2_model is not None:
            self._tier2 = tier2_model
        else:
            from langchain_openai import ChatOpenAI

            self._tier2 = ChatOpenAI(  # type: ignore[assignment]
                model=config.models.evaluator_tier2_model,
                temperature=0,
                timeout=config.safety.model_timeout_seconds,
            ).with_structured_output(EvalResponse)

    def evaluate(
        self,
        task: str,
        output: str,
        context: str | None = None,
        *,
        tool_results: list | None = None,
        record_history: bool = True,
    ) -> EvalResult:
        """Evaluate an LLM output for quality.

        Runs Tier 1 evaluation first. If the score is borderline (within
        uncertainty_range of the threshold) or confidence is low, escalates
        to Tier 2.

        Args:
            task: The original task/instruction that produced the output.
            output: The LLM output to evaluate.
            context: Optional additional context for the evaluator.
            tool_results: Optional list of actual tool execution results. When
                provided, the evaluator can verify the agent's claims against
                what really happened. When None, the evaluation is text-only and
                EvalResult.unverified is set to True. An empty list [] means
                the tool was called and returned nothing — still counts as
                verified (unverified=False). Default None for backward compat.
            record_history: If True (default), updates the rolling history used
                for threshold auto-raise. Pass False for intermediate retry
                evaluations to avoid polluting the rolling average (BUG-049).

        Returns:
            EvalResult with score, pass/fail, feedback, and metadata.
            EvalResult.unverified is True when tool_results is None.
        """
        # Determine whether this evaluation is verified (tool_results provided)
        unverified = tool_results is None

        # Tier 1 evaluation
        try:
            t1_response = self._call_model(self._tier1, task, output, context, tool_results)
        except Exception:
            logger.warning(
                "Tier 1 evaluation failed. Using optimistic fallback (passed=True). "
                "Quality tracking is degraded for this call.",
                exc_info=True,
            )
            return EvalResult(
                score=1.0,
                passed=True,
                feedback="Evaluation unavailable — optimistic fallback applied",
                reasoning="Tier 1 evaluation failed — optimistic fallback applied",
                tier_used=1,
                threshold=self._state.current_threshold,
                confidence=0.5,
                unverified=unverified,
            )
        threshold = self._state.current_threshold

        # Check if escalation to Tier 2 is needed
        if self._should_escalate(t1_response.score, t1_response.confidence):
            try:
                t2_response = self._call_model(self._tier2, task, output, context, tool_results)
                self._state.tier2_escalations += 1

                result = EvalResult(
                    score=t2_response.score,
                    passed=t2_response.score >= threshold,
                    feedback=t2_response.feedback,
                    reasoning=t2_response.reasoning,
                    tier_used=2,
                    threshold=threshold,
                    confidence=t2_response.confidence,
                    unverified=unverified,
                )
            except Exception:
                # Tier 2 failed (network, rate limit, etc.) — fall back to Tier 1.
                # Graceful degradation: a Tier 1 result is better than no result.
                result = EvalResult(
                    score=t1_response.score,
                    passed=t1_response.score >= threshold,
                    feedback=t1_response.feedback,
                    reasoning=t1_response.reasoning,
                    tier_used=1,
                    threshold=threshold,
                    confidence=t1_response.confidence,
                    unverified=unverified,
                )
        else:
            result = EvalResult(
                score=t1_response.score,
                passed=t1_response.score >= threshold,
                feedback=t1_response.feedback,
                reasoning=t1_response.reasoning,
                tier_used=1,
                threshold=threshold,
                confidence=t1_response.confidence,
                unverified=unverified,
            )

        # Update history and check if threshold should auto-raise.
        # BUG-049: skip for intermediate retry evaluations to avoid
        # polluting the rolling average with below-threshold retry scores.
        if record_history:
            self._update_history(result)

        return result

    def record_result(self, result: EvalResult) -> None:
        """Explicitly record an evaluation result into rolling history.

        Used by quality loops that call evaluate() with record_history=False
        on each attempt and then record only the final selected result once
        (BUG-049: prevents retry scores from polluting threshold auto-raise).

        Args:
            result: The EvalResult to record.
        """
        self._update_history(result)

    def _call_model(
        self,
        model: EvalModel,
        task: str,
        output: str,
        context: str | None,
        tool_results: list | None = None,
    ) -> EvalResponse:
        """Call an evaluation model with the structured prompt.

        Args:
            model: The LLM model to invoke.
            task: The original task description.
            output: The LLM output to evaluate.
            context: Optional additional context.
            tool_results: Optional actual tool execution results to include in prompt.

        Returns:
            EvalResponse with score, confidence, reasoning, and feedback.
        """
        from langchain_core.messages import HumanMessage, SystemMessage

        messages = [
            SystemMessage(content=_EVAL_SYSTEM_PROMPT),
            HumanMessage(content=_build_eval_prompt(task, output, context, tool_results=tool_results)),
        ]
        result = model.invoke(messages)

        # BUG-035: with_structured_output() can silently return None when the LLM
        # produces malformed JSON, an empty tool call, or a refusal. Guard here
        # rather than crashing with AttributeError in the caller.
        if result is None:
            logger.warning(
                "Structured output returned None — returning default EvalResponse "
                "(score=0.5, confidence=0.0). This may indicate malformed LLM output."
            )
            return EvalResponse(
                score=0.5,
                confidence=0.0,
                reasoning="Evaluation failed: structured output returned None",
                feedback="Unable to evaluate — retry recommended",
            )

        # BUG-036: OpenAI structured output API does not enforce ge/le field
        # constraints — the LLM can return out-of-range values (e.g., score=1.5).
        # Clamp in-place to prevent Pydantic ValidationError in callers.
        result.score = _clamp_score(result.score)
        result.confidence = _clamp_score(result.confidence)

        return result

    def _should_escalate(self, score: float, confidence: float) -> bool:
        """Determine if Tier 2 evaluation is needed.

        Escalates when:
        1. Score is in the borderline band (asymmetric: full range below threshold,
           zero above — passing scores don't waste Tier 2), OR
        2. Confidence is below the low_confidence_threshold.

        Asymmetric band (BUG-018): `abs(score - threshold)` was symmetric, escalating
        clearly-passing scores. Now: upper bound = threshold + 0 (no above-threshold
        escalation). Bounds clamped to [0.0, 1.0] for edge thresholds near 0 or 1.

        Args:
            score: Tier 1 quality score.
            confidence: Tier 1 confidence in the score.

        Returns:
            True if Tier 2 should be invoked.
        """
        eval_config = self._config.evaluation
        threshold = self._state.current_threshold

        above_margin = eval_config.uncertainty_range * _ESCALATION_ABOVE_THRESHOLD_RATIO
        lower_bound = max(0.0, threshold - eval_config.uncertainty_range)
        upper_bound = min(1.0, threshold + above_margin)
        borderline = lower_bound <= score <= upper_bound
        low_confidence = confidence < eval_config.low_confidence_threshold

        return borderline or low_confidence

    def _update_history(self, result: EvalResult) -> None:
        """Record evaluation result and check for threshold auto-raise.

        Args:
            result: The evaluation result to record.
        """
        self._state.scores_history.append(result.score)
        self._state.total_evaluations += 1

        # Keep history bounded to avoid unbounded growth
        max_history = self._config.evaluation.rolling_window * 3
        if len(self._state.scores_history) > max_history:
            self._state.scores_history = self._state.scores_history[-max_history:]

        self._check_auto_raise()
        self._save_state()

    def _check_auto_raise(self) -> None:
        """Auto-raise threshold if rolling average exceeds threshold + buffer.

        Only raises if:
        1. Enough scores in history (>= rolling_window)
        2. Rolling average > current_threshold + auto_raise_buffer
        3. Current threshold < max_threshold

        The new threshold is rounded to 4 decimal places to prevent IEEE 754
        float accumulation errors (e.g. 0.9000000000000001) from repeated
        additions of auto_raise_increment (BUG-008).
        """
        eval_config = self._config.evaluation
        window = eval_config.rolling_window

        if len(self._state.scores_history) < window:
            return

        recent = self._state.scores_history[-window:]
        rolling_avg = sum(recent) / len(recent)

        threshold = self._state.current_threshold
        target = threshold + eval_config.auto_raise_buffer

        if rolling_avg >= target and threshold < eval_config.max_threshold:
            new_threshold = min(
                round(threshold + eval_config.auto_raise_increment, 4),
                eval_config.max_threshold,
            )
            self._state.current_threshold = new_threshold
            self._state.threshold_raises += 1

    def _load_state(self) -> EvalState:
        """Load evaluation state from file, or create fresh state.

        Returns:
            EvalState loaded from disk, or new state with initial threshold.
        """
        state_path = self._state_path
        if state_path.exists():
            try:
                raw = json.loads(state_path.read_text(encoding="utf-8"))
                return EvalState.model_validate(raw)
            except (json.JSONDecodeError, KeyError, ValueError) as exc:
                # Corrupted or incompatible state file — start fresh.
                # Intentionally narrow: PermissionError/OSError should propagate.
                logger.warning(
                    "Corrupted eval state at %s, starting fresh: %s",
                    state_path,
                    exc,
                )
                return EvalState(
                    current_threshold=self._config.evaluation.initial_threshold,
                )
        return EvalState(
            current_threshold=self._config.evaluation.initial_threshold,
        )

    def _save_state(self) -> None:
        """Persist evaluation state to file.

        BUG-019: file lock prevents concurrent-process data loss.
        BUG-033: atomic_write_json uses rename(2) to prevent partial writes.
        """
        state_path = self._state_path
        lock_path = state_path.with_suffix(state_path.suffix + ".lock")
        with FileLock(lock_path, timeout=10):
            atomic_write_json(state_path, self._state.model_dump())

    @property
    def _state_path(self) -> Path:
        """Path to the evaluation state file."""
        return self._config.namespace_dir / "eval_state.json"

    @property
    def threshold(self) -> float:
        """Current quality threshold (read-only)."""
        return self._state.current_threshold

    @property
    def stats(self) -> dict[str, Any]:
        """Evaluation statistics for monitoring.

        Returns:
            Dict with total evaluations, threshold, escalation rate, etc.
        """
        total = self._state.total_evaluations
        escalations = self._state.tier2_escalations
        return {
            "total_evaluations": total,
            "current_threshold": self._state.current_threshold,
            "threshold_raises": self._state.threshold_raises,
            "tier2_escalations": escalations,
            "tier2_escalation_rate": escalations / total if total > 0 else 0.0,
            "scores_in_history": len(self._state.scores_history),
        }
