"""Model canary — validates model version changes before acceptance.

Before updating .abi/models.yaml, run the model canary to verify no regressions.

Usage:
    python -m abi_evals canary --model-config <path>
"""
from __future__ import annotations

import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from abi_evals.runner import load_eval_suite, run_suite


@dataclass
class RegressionDetail:
    """Details about a single regression."""
    case_id: str
    baseline_status: str
    new_status: str
    baseline_message: str = ""
    new_message: str = ""


@dataclass
class CanaryResult:
    """Result of model canary validation."""
    verdict: str  # PASS | WARN | FAIL
    success_rate_baseline: float
    success_rate_new: float
    success_rate_delta: float
    regression_count: int
    improvement_count: int
    cost_baseline: int
    cost_new: int
    cost_delta: int
    regressions: list[RegressionDetail] = field(default_factory=list)
    improvements: list[RegressionDetail] = field(default_factory=list)
    details: str = ""

    def to_dict(self) -> dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            "verdict": self.verdict,
            "success_rate_baseline": self.success_rate_baseline,
            "success_rate_new": self.success_rate_new,
            "success_rate_delta": self.success_rate_delta,
            "regression_count": self.regression_count,
            "improvement_count": self.improvement_count,
            "cost_baseline": self.cost_baseline,
            "cost_new": self.cost_new,
            "cost_delta": self.cost_delta,
            "regressions": [
                {
                    "case_id": r.case_id,
                    "baseline_status": r.baseline_status,
                    "new_status": r.new_status,
                    "baseline_message": r.baseline_message,
                    "new_message": r.new_message,
                }
                for r in self.regressions
            ],
            "improvements": [
                {
                    "case_id": i.case_id,
                    "baseline_status": i.baseline_status,
                    "new_status": i.new_status,
                }
                for i in self.improvements
            ],
            "details": self.details,
        }


class ModelCanary:
    """Model canary validation gate."""

    def __init__(self, eval_suite_path: str, baseline_results_path: str):
        """Initialize with eval suite and baseline results from current model version.

        Args:
            eval_suite_path: Path to the eval suite JSON file
            baseline_results_path: Path to baseline results JSON (from current model)
        """
        self.eval_suite_path = Path(eval_suite_path)
        self.baseline_results_path = Path(baseline_results_path)

        if not self.eval_suite_path.exists():
            raise FileNotFoundError(f"Eval suite not found: {eval_suite_path}")
        if not self.baseline_results_path.exists():
            raise FileNotFoundError(f"Baseline results not found: {baseline_results_path}")

        # Load eval suite
        self.suite = load_eval_suite(self.eval_suite_path)

        # Load baseline results
        with open(self.baseline_results_path) as f:
            self.baseline_results = json.load(f)

    def run_canary(
        self,
        new_model_config: dict[str, Any],
        _run_suite_func=None,  # For testing: allow injection of custom run_suite
    ) -> CanaryResult:
        """Run the eval suite with new model configuration and compare against baseline.

        Steps:
        a. Load eval suite (golden fixtures from abi-evals)
        b. Run each eval case with the NEW model config (via simulation/replay if available)
        c. Compare results against baseline:
           - success_rate_delta: new_rate - baseline_rate
           - regression_count: cases that passed before but fail now
           - improvement_count: cases that failed before but pass now
           - cost_delta: new_cost - baseline_cost
        d. Verdict:
           - PASS: no regressions AND success_rate_delta >= 0
           - WARN: <=2 regressions AND success_rate_delta >= -0.05
           - FAIL: >2 regressions OR success_rate_delta < -0.05
        e. Return CanaryResult with verdict, metrics, and details

        Args:
            new_model_config: Configuration for the new model to test
            _run_suite_func: (Internal) For testing, inject custom run_suite function

        Returns:
            CanaryResult with verdict and comparison metrics
        """
        # Run eval suite with new model config
        # For now, we simulate by just re-running the suite
        # In production, this would use the new_model_config to call the actual model
        run_suite_func = _run_suite_func or run_suite
        new_results = run_suite_func(
            self.suite,
            evidence_pack=new_model_config.get("evidence_pack"),
            run_id=new_model_config.get("run_id", "canary_test"),
            timestamp=new_model_config.get("timestamp"),
        )

        # Compare results
        baseline_summary = self.baseline_results["summary"]
        new_summary = new_results["summary"]

        # Calculate success rates
        baseline_total = baseline_summary["total"]
        new_total = new_summary["total"]

        if baseline_total == 0 or new_total == 0:
            raise ValueError("Cannot compare results with zero test cases")

        baseline_success_rate = baseline_summary["passed"] / baseline_total
        new_success_rate = new_summary["passed"] / new_total
        success_rate_delta = new_success_rate - baseline_success_rate

        # Find regressions and improvements
        baseline_by_id = {
            r["case_id"]: r for r in self.baseline_results["results"]
        }
        new_by_id = {
            r["case_id"]: r for r in new_results["results"]
        }

        regressions: list[RegressionDetail] = []
        improvements: list[RegressionDetail] = []

        for case_id in baseline_by_id:
            if case_id not in new_by_id:
                continue

            baseline_result = baseline_by_id[case_id]
            new_result = new_by_id[case_id]

            baseline_passed = baseline_result["status"] == "pass"
            new_passed = new_result["status"] == "pass"

            # Regression: passed before, fails now
            if baseline_passed and not new_passed:
                regressions.append(RegressionDetail(
                    case_id=case_id,
                    baseline_status=baseline_result["status"],
                    new_status=new_result["status"],
                    baseline_message=baseline_result.get("message", ""),
                    new_message=new_result.get("message", ""),
                ))

            # Improvement: failed before, passes now
            elif not baseline_passed and new_passed:
                improvements.append(RegressionDetail(
                    case_id=case_id,
                    baseline_status=baseline_result["status"],
                    new_status=new_result["status"],
                    baseline_message=baseline_result.get("message", ""),
                    new_message=new_result.get("message", ""),
                ))

        # Calculate cost delta
        baseline_cost = self.baseline_results.get("cost_summary", {}).get("total_tokens", 0)
        new_cost = new_results.get("cost_summary", {}).get("total_tokens", 0)
        cost_delta = new_cost - baseline_cost

        # Determine verdict
        regression_count = len(regressions)

        if regression_count == 0 and success_rate_delta >= 0:
            verdict = "PASS"
            details = "No regressions detected, success rate maintained or improved"
        elif regression_count <= 2 and success_rate_delta >= -0.05:
            verdict = "WARN"
            details = (
                f"Minor regressions detected ({regression_count} cases), "
                f"success rate delta: {success_rate_delta:.2%}"
            )
        else:
            verdict = "FAIL"
            details = (
                f"Significant regressions detected ({regression_count} cases) or "
                f"success rate dropped significantly (delta: {success_rate_delta:.2%})"
            )

        return CanaryResult(
            verdict=verdict,
            success_rate_baseline=baseline_success_rate,
            success_rate_new=new_success_rate,
            success_rate_delta=success_rate_delta,
            regression_count=regression_count,
            improvement_count=len(improvements),
            cost_baseline=baseline_cost,
            cost_new=new_cost,
            cost_delta=cost_delta,
            regressions=regressions,
            improvements=improvements,
            details=details,
        )
