"""Tests for model_canary module."""
from __future__ import annotations

import json
from pathlib import Path

import pytest

from abi_evals.model_canary import ModelCanary


@pytest.fixture
def eval_suite_path(tmp_path: Path) -> Path:
    """Create a test eval suite."""
    suite = {
        "suite_id": "test_suite",
        "cases": [
            {"case_id": "case_001", "input": 2, "expected": 4, "comparison": "exact"},
            {"case_id": "case_002", "input": 3, "expected": 9, "comparison": "exact"},
            {"case_id": "case_003", "input": 4, "expected": 16, "comparison": "exact"},
            {"case_id": "case_004", "input": 5, "expected": 25, "comparison": "exact"},
            {"case_id": "case_005", "input": 6, "expected": 36, "comparison": "exact"},
        ]
    }
    path = tmp_path / "test_suite.json"
    with open(path, "w") as f:
        json.dump(suite, f)
    return path


@pytest.fixture
def baseline_results_all_pass(tmp_path: Path) -> Path:
    """Create baseline results where all cases pass."""
    results = {
        "schema_version": "eval-result/1.0",
        "suite_id": "test_suite",
        "run_id": "baseline_001",
        "evaluated_at": "2025-01-01T00:00:00Z",
        "results": [
            {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
        ],
        "summary": {
            "total": 5,
            "passed": 5,
            "failed": 0,
            "errors": 0,
            "skipped": 0,
        },
        "cost_summary": {
            "total_tokens": 50,
            "case_token_map": {
                "case_001": 10,
                "case_002": 10,
                "case_003": 10,
                "case_004": 10,
                "case_005": 10,
            },
        },
    }
    path = tmp_path / "baseline_all_pass.json"
    with open(path, "w") as f:
        json.dump(results, f)
    return path


@pytest.fixture
def baseline_results_with_failures(tmp_path: Path) -> Path:
    """Create baseline results with some failures."""
    results = {
        "schema_version": "eval-result/1.0",
        "suite_id": "test_suite",
        "run_id": "baseline_002",
        "evaluated_at": "2025-01-01T00:00:00Z",
        "results": [
            {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_002", "status": "fail", "actual": 8, "message": "Expected 9, got 8", "diff": "Expected 9, got 8", "tokens_used": 10},
            {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
            {"case_id": "case_004", "status": "fail", "actual": 24, "message": "Expected 25, got 24", "diff": "Expected 25, got 24", "tokens_used": 10},
            {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
        ],
        "summary": {
            "total": 5,
            "passed": 3,
            "failed": 2,
            "errors": 0,
            "skipped": 0,
        },
        "cost_summary": {
            "total_tokens": 50,
            "case_token_map": {
                "case_001": 10,
                "case_002": 10,
                "case_003": 10,
                "case_004": 10,
                "case_005": 10,
            },
        },
    }
    path = tmp_path / "baseline_with_failures.json"
    with open(path, "w") as f:
        json.dump(results, f)
    return path


def test_canary_pass_no_regressions(eval_suite_path: Path, baseline_results_all_pass: Path):
    """Test PASS verdict when no regressions occur."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_all_pass),
    )

    # Mock run_suite to return results identical to baseline
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "canary_test",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
            ],
            "summary": {"total": 5, "passed": 5, "failed": 0, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 50,
                "case_token_map": {
                    "case_001": 10, "case_002": 10, "case_003": 10, "case_004": 10, "case_005": 10
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)

    # Verify PASS verdict
    assert result.verdict == "PASS"
    assert result.regression_count == 0
    assert result.success_rate_delta >= 0
    assert "No regressions detected" in result.details


def test_canary_pass_with_improvements(eval_suite_path: Path, baseline_results_with_failures: Path):
    """Test PASS verdict when improvements occur without regressions."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_with_failures),
    )

    # Mock run_suite to return all passing results
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "canary_test",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
            ],
            "summary": {"total": 5, "passed": 5, "failed": 0, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 50,
                "case_token_map": {
                    "case_001": 10, "case_002": 10, "case_003": 10, "case_004": 10, "case_005": 10
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)

    # Verify PASS verdict
    assert result.verdict == "PASS"
    assert result.regression_count == 0
    assert result.improvement_count == 2  # case_002 and case_004 now pass
    assert result.success_rate_baseline == 0.6  # 3/5
    assert result.success_rate_new == 1.0  # 5/5
    assert result.success_rate_delta == 0.4  # +40%


def test_canary_warn_minor_regressions(eval_suite_path: Path, baseline_results_all_pass: Path, tmp_path: Path):
    """Test WARN verdict with 1-2 regressions and success rate drop < 5%."""
    # Create baseline with 20 cases for better granularity
    baseline_20 = {
        "schema_version": "eval-result/1.0",
        "suite_id": "test_suite_20",
        "run_id": "baseline_20",
        "evaluated_at": "2025-01-01T00:00:00Z",
        "results": [
            {"case_id": f"case_{i:03d}", "status": "pass", "actual": i, "message": "", "diff": "", "tokens_used": 10}
            for i in range(20)
        ],
        "summary": {"total": 20, "passed": 20, "failed": 0, "errors": 0, "skipped": 0},
        "cost_summary": {
            "total_tokens": 200,
            "case_token_map": {f"case_{i:03d}": 10 for i in range(20)},
        },
    }

    baseline_path = tmp_path / "baseline_20.json"
    with open(baseline_path, "w") as f:
        json.dump(baseline_20, f)

    suite_20 = {
        "suite_id": "test_suite_20",
        "cases": [
            {"case_id": f"case_{i:03d}", "input": i, "expected": i, "comparison": "exact"}
            for i in range(20)
        ]
    }
    suite_path = tmp_path / "suite_20.json"
    with open(suite_path, "w") as f:
        json.dump(suite_20, f)

    canary = ModelCanary(
        eval_suite_path=str(suite_path),
        baseline_results_path=str(baseline_path),
    )

    # Create new results with 1 regression but success rate stays > 95% (delta > -5%)
    # 20/21 pass = 95.24%, delta = -4.76%
    # Add one more case to baseline and new results
    baseline_21 = {
        "schema_version": "eval-result/1.0",
        "suite_id": "test_suite_20",
        "run_id": "baseline_20",
        "evaluated_at": "2025-01-01T00:00:00Z",
        "results": [
            {"case_id": f"case_{i:03d}", "status": "pass", "actual": i, "message": "", "diff": "", "tokens_used": 10}
            for i in range(21)
        ],
        "summary": {"total": 21, "passed": 21, "failed": 0, "errors": 0, "skipped": 0},
        "cost_summary": {
            "total_tokens": 210,
            "case_token_map": {f"case_{i:03d}": 10 for i in range(21)},
        },
    }
    baseline_path_21 = tmp_path / "baseline_21.json"
    with open(baseline_path_21, "w") as f:
        json.dump(baseline_21, f)

    suite_21 = {
        "suite_id": "test_suite_20",
        "cases": [
            {"case_id": f"case_{i:03d}", "input": i, "expected": i, "comparison": "exact"}
            for i in range(21)
        ]
    }
    suite_path_21 = tmp_path / "suite_21.json"
    with open(suite_path_21, "w") as f:
        json.dump(suite_21, f)

    canary_21 = ModelCanary(
        eval_suite_path=str(suite_path_21),
        baseline_results_path=str(baseline_path_21),
    )

    def mock_run_suite(*args, **kwargs):
        results = [
            {"case_id": f"case_{i:03d}", "status": "pass", "actual": i, "message": "", "diff": "", "tokens_used": 10}
            for i in range(21)
        ]
        results[0]["status"] = "fail"  # 1 regression: 20/21 = 95.24%
        results[0]["message"] = "Regression"

        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite_20",
            "run_id": "new_001",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": results,
            "summary": {"total": 21, "passed": 20, "failed": 1, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 210,
                "case_token_map": {f"case_{i:03d}": 10 for i in range(21)},
            },
        }

    result = canary_21.run_canary({}, _run_suite_func=mock_run_suite)

    # Verify WARN verdict: 1 regression and delta = -4.76% (above -5% threshold)
    assert result.verdict == "WARN"
    assert result.regression_count == 1
    assert result.success_rate_baseline == 1.0  # 21/21
    assert abs(result.success_rate_new - (20/21)) < 0.001  # 20/21 = 95.24%
    assert result.success_rate_delta > -0.05  # Better than -5%
    assert len(result.regressions) == 1
    assert result.regressions[0].case_id == "case_000"
    assert result.regressions[0].baseline_status == "pass"
    assert result.regressions[0].new_status == "fail"


def test_canary_fail_many_regressions(eval_suite_path: Path, baseline_results_all_pass: Path):
    """Test FAIL verdict with 3+ regressions."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_all_pass),
    )

    # Create new results with 3 regressions
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "new_002",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_002", "status": "fail", "actual": 8, "message": "Regression 1", "diff": "Regression 1", "tokens_used": 10},
                {"case_id": "case_003", "status": "fail", "actual": 15, "message": "Regression 2", "diff": "Regression 2", "tokens_used": 10},
                {"case_id": "case_004", "status": "fail", "actual": 24, "message": "Regression 3", "diff": "Regression 3", "tokens_used": 10},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
            ],
            "summary": {"total": 5, "passed": 2, "failed": 3, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 50,
                "case_token_map": {
                    "case_001": 10, "case_002": 10, "case_003": 10, "case_004": 10, "case_005": 10
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)

    # Verify FAIL verdict
    assert result.verdict == "FAIL"
    assert result.regression_count == 3
    assert result.success_rate_baseline == 1.0  # 5/5
    assert result.success_rate_new == 0.4  # 2/5
    assert result.success_rate_delta == -0.6  # -60%
    assert len(result.regressions) == 3
    assert "Significant regressions" in result.details


def test_canary_fail_large_success_rate_drop(eval_suite_path: Path, baseline_results_all_pass: Path):
    """Test FAIL verdict when success rate drops significantly (>5%)."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_all_pass),
    )

    # Create new results with 1 regression but represents >5% drop
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "new_003",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "fail", "actual": 3, "message": "Regression", "diff": "Regression", "tokens_used": 10},
                {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
            ],
            "summary": {"total": 5, "passed": 4, "failed": 1, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 50,
                "case_token_map": {
                    "case_001": 10, "case_002": 10, "case_003": 10, "case_004": 10, "case_005": 10
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)

    # Success rate delta is -20%, which is < -5%, so this should FAIL
    # WARN: regression_count <= 2 AND success_rate_delta >= -0.05
    # FAIL: >2 regressions OR success_rate_delta < -0.05
    # With 1 regression and -20% delta, delta is -0.2 < -0.05, so FAIL
    assert result.verdict == "FAIL"
    assert result.regression_count == 1
    assert abs(result.success_rate_delta - (-0.2)) < 0.001  # -20% (use floating point comparison)


def test_canary_cost_comparison(eval_suite_path: Path, baseline_results_all_pass: Path):
    """Test that cost delta is computed correctly."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_all_pass),
    )

    # Create new results with higher token usage
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "new_004",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 15},
                {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 20},
                {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 18},
                {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 12},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 25},
            ],
            "summary": {"total": 5, "passed": 5, "failed": 0, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 90,
                "case_token_map": {
                    "case_001": 15, "case_002": 20, "case_003": 18, "case_004": 12, "case_005": 25
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)

    # Verify cost comparison
    assert result.cost_baseline == 50
    assert result.cost_new == 90
    assert result.cost_delta == 40  # +40 tokens
    assert result.verdict == "PASS"  # No regressions


def test_canary_to_dict(eval_suite_path: Path, baseline_results_all_pass: Path):
    """Test CanaryResult.to_dict() serialization."""
    canary = ModelCanary(
        eval_suite_path=str(eval_suite_path),
        baseline_results_path=str(baseline_results_all_pass),
    )

    # Mock run_suite to return passing results
    def mock_run_suite(*args, **kwargs):
        return {
            "schema_version": "eval-result/1.0",
            "suite_id": "test_suite",
            "run_id": "canary_test",
            "evaluated_at": "2025-01-02T00:00:00Z",
            "results": [
                {"case_id": "case_001", "status": "pass", "actual": 4, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_002", "status": "pass", "actual": 9, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_003", "status": "pass", "actual": 16, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_004", "status": "pass", "actual": 25, "message": "", "diff": "", "tokens_used": 10},
                {"case_id": "case_005", "status": "pass", "actual": 36, "message": "", "diff": "", "tokens_used": 10},
            ],
            "summary": {"total": 5, "passed": 5, "failed": 0, "errors": 0, "skipped": 0},
            "cost_summary": {
                "total_tokens": 50,
                "case_token_map": {
                    "case_001": 10, "case_002": 10, "case_003": 10, "case_004": 10, "case_005": 10
                },
            },
        }

    result = canary.run_canary({}, _run_suite_func=mock_run_suite)
    result_dict = result.to_dict()

    # Verify all fields are present
    assert "verdict" in result_dict
    assert "success_rate_baseline" in result_dict
    assert "success_rate_new" in result_dict
    assert "success_rate_delta" in result_dict
    assert "regression_count" in result_dict
    assert "improvement_count" in result_dict
    assert "cost_baseline" in result_dict
    assert "cost_new" in result_dict
    assert "cost_delta" in result_dict
    assert "regressions" in result_dict
    assert "improvements" in result_dict
    assert "details" in result_dict

    # Verify JSON serializable
    json_str = json.dumps(result_dict)
    assert json_str is not None


def test_canary_file_not_found():
    """Test error handling when files don't exist."""
    with pytest.raises(FileNotFoundError):
        ModelCanary(
            eval_suite_path="nonexistent_suite.json",
            baseline_results_path="nonexistent_baseline.json",
        )
