# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Licensed under the MIT License - see LICENSE file for details

"""Tests for the eval pipeline — baseline suite, evaluation framework, and CI runner."""

import json
from pathlib import Path

import pytest

BASELINE_PATH = Path(__file__).parent / "evals" / "baseline.json"

REQUIRED_CATEGORIES = {"conversation", "tool_use", "multi_step", "safety", "personality"}
REQUIRED_SEVERITIES = {"critical", "high", "medium", "low"}


# ── baseline.json structural tests ──────────────────────────────────


class TestBaselineSuite:
    """Validate the golden baseline test suite."""

    @pytest.fixture(autouse=True)
    def _load(self):
        self.data = json.loads(BASELINE_PATH.read_text())
        self.tests = self.data["tests"]

    def test_valid_json_with_enough_tests(self):
        assert len(self.tests) >= 25

    def test_required_fields(self):
        for t in self.tests:
            assert "id" in t, f"Missing 'id': {t.get('name', t.get('input', '?'))}"
            assert "input" in t, f"Missing 'input': {t.get('id', '?')}"
            assert "category" in t, f"Missing 'category': {t.get('id', '?')}"

    def test_all_categories_represented(self):
        categories = {t["category"] for t in self.tests}
        missing = REQUIRED_CATEGORIES - categories
        assert not missing, f"Missing categories: {missing}"

    def test_all_severity_levels_used(self):
        severities = {t.get("severity", "medium") for t in self.tests}
        missing = REQUIRED_SEVERITIES - severities
        assert not missing, f"Missing severity levels: {missing}"


# ── evaluation.py unit tests ────────────────────────────────────────


class TestEvaluationFramework:
    """Test evaluation module functions and classes."""

    def test_load_test_suite_returns_correct_structure(self):
        from familiar.core.evaluation import load_test_suite

        suite = load_test_suite(BASELINE_PATH)
        assert "name" in suite
        assert "tests" in suite
        assert len(suite["tests"]) >= 25
        # Should be TestCase objects, not dicts
        from familiar.core.evaluation import TestCase

        assert isinstance(suite["tests"][0], TestCase)

    def test_create_test_case_factory(self):
        from familiar.core.evaluation import TestCase, create_test_case

        tc = create_test_case(
            input="Hello",
            expected_contains=["hi"],
            severity="medium",
            category="conversation",
        )
        assert isinstance(tc, TestCase)
        assert tc.input == "Hello"
        assert tc.expected_contains == ["hi"]
        assert tc.category == "conversation"

    def test_dataset_builder_add_from_trace(self):
        from familiar.core.evaluation import EvalDatasetBuilder

        builder = EvalDatasetBuilder()
        builder.add_from_trace(
            user_input="What time is it?",
            agent_response="It is 3pm.",
            tools_called=["get_time"],
            success=True,
        )
        assert len(builder.pending_cases) == 1
        assert builder.pending_cases[0]["input"] == "What time is it?"

    def test_dataset_builder_build_dataset_respects_min_cases(self, tmp_path):
        from familiar.core.evaluation import EvalDatasetBuilder

        builder = EvalDatasetBuilder(output_dir=tmp_path)
        builder.add_from_trace(
            user_input="Hi",
            agent_response="Hello!",
            tools_called=[],
            success=True,
        )
        # min_cases=10 (default) should return None with only 1 case
        result = builder.build_dataset(name="test", min_cases=10)
        assert result is None

        # min_cases=1 should succeed
        result = builder.build_dataset(name="test", min_cases=1)
        assert result is not None
        assert result.exists()

    def test_contains_scorer(self):
        from familiar.core.evaluation import ContainsScorer, TestCase

        scorer = ContainsScorer()
        tc = TestCase(input="hi", expected_contains=["hello", "world"])

        score, passed, msg = scorer.score(tc, "Hello world!", {})
        assert passed
        assert score == 1.0

        score, passed, msg = scorer.score(tc, "Goodbye!", {})
        assert not passed
        assert score < 1.0

    def test_not_contains_scorer(self):
        from familiar.core.evaluation import NotContainsScorer, TestCase

        scorer = NotContainsScorer()
        tc = TestCase(input="hi", expected_not_contains=["error", "fail"])

        score, passed, msg = scorer.score(tc, "All good!", {})
        assert passed
        assert score == 1.0

        score, passed, msg = scorer.score(tc, "An error occurred", {})
        assert not passed
        assert score == 0.0

    def test_safety_scorer(self):
        from familiar.core.evaluation import SafetyScorer, TestCase

        scorer = SafetyScorer()
        tc = TestCase(input="hi")

        score, passed, msg = scorer.score(tc, "Here is the information you requested.", {})
        assert passed
        assert score == 1.0


# ── ci_eval.py tests ────────────────────────────────────────────────


class TestCIEval:
    """Test CI evaluation runner and threshold checking."""

    def test_check_thresholds_detects_pass_rate_violation(self):
        from familiar.core.ci_eval import EvalThresholds, check_thresholds

        thresholds = EvalThresholds(min_pass_rate=0.90)
        report = {"pass_rate": 0.80, "critical_failures": 0, "high_failures": 0}
        result = check_thresholds(report, thresholds)
        assert not result.passed
        assert any("pass rate" in v.lower() for v in result.violations)

    def test_check_thresholds_detects_critical_failure_violation(self):
        from familiar.core.ci_eval import EvalThresholds, check_thresholds

        thresholds = EvalThresholds(fail_on_any_critical=True)
        report = {"pass_rate": 1.0, "critical_failures": 1, "high_failures": 0}
        result = check_thresholds(report, thresholds)
        assert not result.passed
        assert any("critical" in v.lower() for v in result.violations)

    def test_ci_eval_runner_dry_run(self, tmp_path):
        from familiar.core.ci_eval import CIConfig, CIEvalRunner, OutputFormat

        config = CIConfig(
            suite_path=str(BASELINE_PATH),
            output_dir=str(tmp_path),
            dry_run=True,
            output_formats=[OutputFormat.JSON],
        )
        runner = CIEvalRunner(config)
        result = runner.run()
        assert result.success
        assert result.exit_code == 0
        assert result.report_dict["pass_rate"] > 0
