# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

import json
import uuid
from datetime import UTC, datetime

import pytest
from pydantic import ValidationError

from llama_verifications.schemas.report_v1 import (
    BenchmarkResult,
    Benchmarks,
    BenchmarksSummary,
    EvalTestCase,
    FunctionalTests,
    FunctionalTestsSummary,
    ReportV1,
)


class TestEvalTestCase:
    def test_valid_test_case(self):
        """Test creating a valid test case."""
        test_case = EvalTestCase(
            name="test_example::test_function",
            status="passed",
            duration_ms=150.5,
            error=None,
        )
        assert test_case.name == "test_example::test_function"
        assert test_case.status == "passed"
        assert test_case.duration_ms == 150.5
        assert test_case.error is None

    def test_invalid_status(self):
        """Test that invalid status raises validation error."""
        with pytest.raises(ValidationError):
            EvalTestCase(
                name="test_example::test_function",
                status="invalid_status",
            )

    def test_negative_duration(self):
        """Test that negative duration raises validation error."""
        with pytest.raises(ValidationError):
            EvalTestCase(
                name="test_example::test_function",
                status="passed",
                duration_ms=-10.0,
            )

    def test_failed_test_with_error(self):
        """Test failed test case with error message."""
        test_case = EvalTestCase(
            name="test_example::test_function",
            status="failed",
            duration_ms=50.0,
            error="AssertionError: Expected 5, got 3",
        )
        assert test_case.status == "failed"
        assert test_case.error == "AssertionError: Expected 5, got 3"


class TestFunctionalTestsSummary:
    def test_valid_summary(self):
        """Test creating a valid functional tests summary."""
        summary = FunctionalTestsSummary(
            total=100,
            passed=95,
            failed=3,
            xfailed=1,
            xpassed=0,
            skipped=1,
            pass_rate=95.0,
        )
        assert summary.total == 100
        assert summary.passed == 95
        assert summary.pass_rate == 95.0

    def test_auto_calculate_pass_rate(self):
        """Test that pass rate is auto-calculated when not provided."""
        summary = FunctionalTestsSummary(
            total=100,
            passed=80,
            failed=20,
            xfailed=0,
            xpassed=0,
            skipped=0,
        )
        assert summary.pass_rate == 80.0

    def test_zero_total_pass_rate(self):
        """Test pass rate calculation with zero total."""
        summary = FunctionalTestsSummary(
            total=0,
            passed=0,
            failed=0,
            xfailed=0,
            xpassed=0,
            skipped=0,
        )
        assert summary.pass_rate == 0.0

    def test_invalid_pass_rate_range(self):
        """Test that pass rate outside 0-100 range raises validation error."""
        with pytest.raises(ValidationError):
            FunctionalTestsSummary(
                total=100,
                passed=95,
                failed=5,
                xfailed=0,
                xpassed=0,
                skipped=0,
                pass_rate=150.0,  # Invalid: > 100
            )


class TestBenchmarkResult:
    def test_valid_benchmark_result(self):
        """Test creating a valid benchmark result."""
        benchmark = BenchmarkResult(
            name="mmlu",
            topline_metric_name="accuracy",
            metrics={"accuracy": 0.78, "f1": 0.75},
            subset_metrics={
                "humanities": {"accuracy": 0.81, "f1": 0.78},
                "stem": {"accuracy": 0.75, "f1": 0.72},
            },
            config={"n_shot": 5, "temperature": 0.0},
        )
        assert benchmark.name == "mmlu"
        assert benchmark.topline_metric_name == "accuracy"
        assert benchmark.metrics["accuracy"] == 0.78
        assert benchmark.subset_metrics["humanities"]["accuracy"] == 0.81

    def test_minimal_benchmark_result(self):
        """Test creating benchmark result with minimal required fields."""
        benchmark = BenchmarkResult(
            name="simple_benchmark",
            topline_metric_name="score",
            metrics={"score": 0.85},
        )
        assert benchmark.name == "simple_benchmark"
        assert benchmark.subset_metrics is None
        assert benchmark.config is None


class TestReportV1:
    def test_valid_report(self):
        """Test creating a valid report."""
        run_id = uuid.uuid4()
        generated_at = datetime.now(UTC)

        functional_tests = FunctionalTests(
            summary=FunctionalTestsSummary(total=10, passed=9, failed=1, xfailed=0, xpassed=0, skipped=0),
            cases=[
                EvalTestCase(name="test1", status="passed"),
                EvalTestCase(name="test2", status="failed", error="Test error"),
            ],
        )

        benchmarks = Benchmarks(
            summary=BenchmarksSummary(total=2),
            benchmarks=[
                BenchmarkResult(
                    name="mmlu",
                    topline_metric_name="accuracy",
                    metrics={"accuracy": 0.78},
                ),
                BenchmarkResult(
                    name="hellaswag",
                    topline_metric_name="accuracy",
                    metrics={"accuracy": 0.85},
                ),
            ],
        )

        report = ReportV1(
            generated_at=generated_at,
            run_id=run_id,
            model="Llama-3-70B",
            model_checkpoint="pretrain-ckpt-20240530",
            provider="Groq",
            functional_tests=functional_tests,
            benchmarks=benchmarks,
        )

        assert report.schema_version == "1.0"
        assert report.run_id == run_id
        assert report.model == "Llama-3-70B"
        assert report.provider == "Groq"
        assert report.functional_tests.summary.total == 10
        assert report.benchmarks.summary.total == 2

    def test_report_with_metadata(self):
        """Test creating report with metadata."""
        report = ReportV1(
            generated_at=datetime.now(UTC),
            run_id=uuid.uuid4(),
            model="Llama-3-70B",
            model_checkpoint="pretrain-ckpt-20240530",
            provider="Groq",
            metadata={
                "provider_endpoint": "https://api.groq.com/v1",
                "eval_suite_version": "d4e1c9b7",
                "custom_field": "custom_value",
            },
            functional_tests=FunctionalTests(
                summary=FunctionalTestsSummary(total=0, passed=0, failed=0, xfailed=0, xpassed=0, skipped=0),
                cases=[],
            ),
            benchmarks=Benchmarks(
                summary=BenchmarksSummary(total=0),
                benchmarks=[],
            ),
        )

        assert report.metadata["provider_endpoint"] == "https://api.groq.com/v1"
        assert report.metadata["eval_suite_version"] == "d4e1c9b7"
        assert report.metadata["custom_field"] == "custom_value"

    def test_to_json_dict(self):
        """Test converting report to JSON dictionary."""
        run_id = uuid.uuid4()
        generated_at = datetime.now(UTC)

        report = ReportV1(
            generated_at=generated_at,
            run_id=run_id,
            model="Llama-3-70B",
            model_checkpoint="pretrain-ckpt-20240530",
            provider="Groq",
            functional_tests=FunctionalTests(
                summary=FunctionalTestsSummary(total=1, passed=1, failed=0, xfailed=0, xpassed=0, skipped=0),
                cases=[EvalTestCase(name="test1", status="passed")],
            ),
            benchmarks=Benchmarks(
                summary=BenchmarksSummary(total=0),
                benchmarks=[],
            ),
        )

        json_dict = report.to_json_dict()

        assert json_dict["schema_version"] == "1.0"
        assert json_dict["run_id"] == str(run_id)
        assert json_dict["model"] == "Llama-3-70B"
        assert json_dict["functional_tests"]["summary"]["total"] == 1
        assert json_dict["benchmarks"]["summary"]["total"] == 0

        # Test that it's JSON serializable
        json_str = json.dumps(json_dict)
        assert isinstance(json_str, str)

    def test_missing_required_fields(self):
        """Test that missing required fields raise validation errors."""
        with pytest.raises(ValidationError):
            ReportV1(
                # Missing required fields
                model="Llama-3-70B",
            )

    def test_invalid_run_id(self):
        """Test that invalid UUID raises validation error."""
        with pytest.raises(ValidationError):
            ReportV1(
                generated_at=datetime.now(UTC),
                run_id="not-a-uuid",  # Invalid UUID
                model="Llama-3-70B",
                model_checkpoint="pretrain-ckpt-20240530",
                provider="Groq",
                functional_tests=FunctionalTests(
                    summary=FunctionalTestsSummary(total=0, passed=0, failed=0, xfailed=0, xpassed=0, skipped=0),
                    cases=[],
                ),
                benchmarks=Benchmarks(
                    summary=BenchmarksSummary(total=0),
                    benchmarks=[],
                ),
            )


class TestJSONSchemaCompliance:
    def test_example_report_matches_schema(self):
        """Test that the example from the schema document is valid."""
        example_data = {
            "schema_version": "1.0",
            "generated_at": "2025-06-05T21:07:00Z",
            "run_id": "3fb4b130-c9f7-4de4-9b9e-1e0f9655e1a2",
            "model": "Llama-3-70B",
            "model_checkpoint": "pretrain-ckpt-20240530",
            "provider": "Groq",
            "metadata": {
                "provider_endpoint": "https://api.groq.com/v1",
                "eval_suite_version": "d4e1c9b7",
            },
            "functional_tests": {
                "summary": {
                    "total": 128,
                    "passed": 127,
                    "failed": 0,
                    "xfailed": 0,
                    "xpassed": 0,
                    "skipped": 1,
                    "pass_rate": 99.2,
                },
                "cases": [
                    {
                        "name": "tests.chat.test_basic::test_ok",
                        "status": "passed",
                        "duration_ms": 87,
                    },
                    {
                        "name": "tests.chat.test_streaming::test_token_stream",
                        "status": "skipped",
                        "duration_ms": 0,
                    },
                ],
            },
            "benchmarks": {
                "summary": {"total": 3},
                "benchmarks": [
                    {
                        "name": "mmlu",
                        "topline_metric_name": "accuracy",
                        "metrics": {"accuracy": 0.78},
                        "subset_metrics": {
                            "humanities": {"accuracy": 0.81},
                            "stem": {"accuracy": 0.75},
                        },
                        "config": {"n_shot": 5, "temperature": 0},
                    },
                    {
                        "name": "truthful_qa_mc",
                        "topline_metric_name": "accuracy",
                        "metrics": {"accuracy": 0.59},
                    },
                    {
                        "name": "mt_bench",
                        "topline_metric_name": "score",
                        "metrics": {"score": 7.2},
                    },
                ],
            },
        }

        # This should not raise any validation errors
        report = ReportV1(**example_data)

        assert report.schema_version == "1.0"
        assert report.model == "Llama-3-70B"
        assert report.functional_tests.summary.total == 128
        assert report.benchmarks.summary.total == 3
        assert len(report.benchmarks.benchmarks) == 3
